diff options
author | Sidath Senanayake <sidaths@google.com> | 2020-06-18 09:26:13 +0200 |
---|---|---|
committer | Sidath Senanayake <sidaths@google.com> | 2020-06-18 09:26:13 +0200 |
commit | bc3c01e61c8ce9783a8ab091053905effcae12de (patch) | |
tree | 43f9f48736f7259d8cc996b11eed003dc80d5bef /mali_kbase | |
parent | b64f568f943e567534694cc993270adca96dcd06 (diff) | |
download | gpu-bc3c01e61c8ce9783a8ab091053905effcae12de.tar.gz |
Mali Valhall DDK r25p0 KMD
Provenance:
395644cb0 (collaborate/EAC/v_r25p0)
VX504X08X-BU-00000-r25p0-01eac0 - Android DDK
VX504X08X-BU-60000-r25p0-01eac0 - Android Document Bundle
Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: I2cffddb42a554696d45b7f65c7bae8827a71341f
Diffstat (limited to 'mali_kbase')
87 files changed, 6009 insertions, 2636 deletions
diff --git a/mali_kbase/Kbuild b/mali_kbase/Kbuild index 7abe8d3..192ac06 100644 --- a/mali_kbase/Kbuild +++ b/mali_kbase/Kbuild @@ -21,7 +21,7 @@ # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= "r24p0-01rel0" +MALI_RELEASE_NAME ?= "r25p0-01eac0" # Paths required for build KBASE_PATH = $(src) @@ -73,6 +73,7 @@ subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UM SRC := \ context/mali_kbase_context.c \ + debug/mali_kbase_debug_ktrace.c \ device/mali_kbase_device.c \ mali_kbase_cache_policy.c \ mali_kbase_mem.c \ @@ -81,7 +82,6 @@ SRC := \ mali_kbase_ctx_sched.c \ mali_kbase_jm.c \ mali_kbase_gpuprops.c \ - mali_kbase_js.c \ mali_kbase_pm.c \ mali_kbase_config.c \ mali_kbase_vinstr.c \ @@ -120,6 +120,7 @@ SRC := \ ifeq ($(MALI_USE_CSF),1) SRC += \ + debug/backend/mali_kbase_debug_ktrace_csf.c \ device/backend/mali_kbase_device_csf.c \ gpu/backend/mali_kbase_gpu_fault_csf.c \ tl/backend/mali_kbase_timeline_csf.c \ @@ -132,7 +133,9 @@ else mali_kbase_event.c \ mali_kbase_jd.c \ mali_kbase_jd_debugfs.c \ + mali_kbase_js.c \ mali_kbase_js_ctx_attr.c \ + debug/backend/mali_kbase_debug_ktrace_jm.c \ device/backend/mali_kbase_device_jm.c \ gpu/backend/mali_kbase_gpu_fault_jm.c \ tl/backend/mali_kbase_timeline_jm.c \ @@ -153,7 +156,8 @@ ifeq ($(MALI_CUSTOMER_RELEASE),0) endif -ccflags-y += -I$(KBASE_PATH) +ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \ + -I$(KBASE_PATH)/debug/backend # Tell the Linux build system from which .o file to create the kernel module obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o @@ -179,6 +183,12 @@ else # empty endif +ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) + include $(src)/arbiter/Kbuild +else +# empty +endif + mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \ mali_kbase_dma_fence.o \ mali_kbase_fence.o diff --git a/mali_kbase/Kconfig b/mali_kbase/Kconfig index a46305d..58a5b0b 100644 --- a/mali_kbase/Kconfig +++ b/mali_kbase/Kconfig @@ -49,6 +49,7 @@ config MALI_MIDGARD_DVFS config MALI_MIDGARD_ENABLE_TRACE bool "Enable kbase tracing" depends on MALI_MIDGARD + default y if MALI_DEBUG default n help Enables tracing in kbase. Trace log available through @@ -84,6 +85,17 @@ config MALI_PLATFORM_NAME include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must exist. +config MALI_ARBITER_SUPPORT + bool "Enable arbiter support for Mali" + depends on MALI_MIDGARD + default n + help + Enable support for the arbiter interface in the driver. + This allows an external arbiter to manage driver access + to GPU hardware in a virtualized environment + + If unsure, say N. + # MALI_EXPERT configuration options menuconfig MALI_EXPERT @@ -156,6 +168,7 @@ config MALI_ERROR_INJECT config MALI_SYSTEM_TRACE bool "Enable system event tracing support" depends on MALI_MIDGARD && MALI_EXPERT + default y if MALI_DEBUG default n help Choose this option to enable system trace events for each @@ -240,6 +253,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE tree using the property, opp-mali-errata-1485982. Otherwise the slowest clock will be selected. +config MALI_GEM5_BUILD + bool "Enable build of Mali kernel driver for GEM5" + depends on MALI_MIDGARD + default n + help + This option is to do a Mali GEM5 build. + If unsure, say N. + # Instrumentation options. config MALI_JOB_DUMP diff --git a/mali_kbase/Mconfig b/mali_kbase/Mconfig index 27399fa..b137793 100644 --- a/mali_kbase/Mconfig +++ b/mali_kbase/Mconfig @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2012-2019 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -40,6 +40,7 @@ config MALI_MIDGARD_DVFS config MALI_MIDGARD_ENABLE_TRACE bool "Enable kbase tracing" depends on MALI_MIDGARD + default y if MALI_DEBUG default n help Enables tracing in kbase. Trace log available through @@ -81,6 +82,17 @@ config MALI_PLATFORM_NAME When PLATFORM_CUSTOM is set, this needs to be set manually to pick up the desired platform files. +config MALI_ARBITER_SUPPORT + bool "Enable arbiter support for Mali" + depends on MALI_MIDGARD + default n + help + Enable support for the arbiter interface in the driver. + This allows an external arbiter to manage driver access + to GPU hardware in a virtualized environment + + If unsure, say N. + # MALI_EXPERT configuration options menuconfig MALI_EXPERT @@ -167,10 +179,11 @@ config MALI_ERROR_INJECT config MALI_SYSTEM_TRACE bool "Enable system event tracing support" depends on MALI_MIDGARD && MALI_EXPERT + default y if MALI_DEBUG default n help Choose this option to enable system trace events for each - kbase event. This is typically used for debugging but has + kbase event. This is typically used for debugging but has minimal overhead when not in use. Enable only if you know what you are doing. @@ -249,6 +262,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE tree using the property, opp-mali-errata-1485982. Otherwise the slowest clock will be selected. +config MALI_GEM5_BUILD + bool "Enable build of Mali kernel driver for GEM5" + depends on MALI_MIDGARD + default n + help + This option is to do a Mali GEM5 build. + If unsure, say N. + # Instrumentation options. # config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig. diff --git a/mali_kbase/arbiter/Kbuild b/mali_kbase/arbiter/Kbuild new file mode 100644 index 0000000..98e47be --- /dev/null +++ b/mali_kbase/arbiter/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + arbiter/mali_kbase_arbif.o \ + arbiter/mali_kbase_arbiter_pm.o diff --git a/mali_kbase/arbiter/mali_kbase_arbif.c b/mali_kbase/arbiter/mali_kbase_arbif.c new file mode 100644 index 0000000..d193cb9 --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbif.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_arbif.c + * Mali arbiter interface APIs to share GPU between Virtual Machines + */ + +#include <mali_kbase.h> +#include "mali_kbase_arbif.h" +#include <tl/mali_kbase_tracepoints.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include "mali_kbase_arbiter_interface.h" + +static void on_gpu_stop(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED(kbdev, kbdev); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); +} + +static void on_gpu_granted(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED(kbdev, kbdev); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); +} + +static void on_gpu_lost(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); +} + +int kbase_arbif_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_OF + struct arbiter_if_arb_vm_ops ops; + struct arbiter_if_dev *arb_if; + struct device_node *arbiter_if_node; + struct platform_device *pdev; + int err; + + dev_dbg(kbdev->dev, "%s\n", __func__); + + arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, + "arbiter_if", 0); + if (!arbiter_if_node) { + dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); + /* no arbiter interface defined in device tree */ + kbdev->arb.arb_dev = NULL; + kbdev->arb.arb_if = NULL; + return 0; + } + + pdev = of_find_device_by_node(arbiter_if_node); + if (!pdev) { + dev_err(kbdev->dev, "Failed to find arbiter_if device\n"); + return -EPROBE_DEFER; + } + + if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { + dev_err(kbdev->dev, "arbiter_if driver not available\n"); + return -EPROBE_DEFER; + } + kbdev->arb.arb_dev = &pdev->dev; + arb_if = platform_get_drvdata(pdev); + if (!arb_if) { + dev_err(kbdev->dev, "arbiter_if driver not ready\n"); + module_put(pdev->dev.driver->owner); + return -EPROBE_DEFER; + } + + kbdev->arb.arb_if = arb_if; + ops.arb_vm_gpu_stop = on_gpu_stop; + ops.arb_vm_gpu_granted = on_gpu_granted; + ops.arb_vm_gpu_lost = on_gpu_lost; + + /* register kbase arbiter_if callbacks */ + if (arb_if->vm_ops.vm_arb_register_dev) { + err = arb_if->vm_ops.vm_arb_register_dev(arb_if, + kbdev->dev, &ops); + if (err) { + dev_err(kbdev->dev, "Arbiter registration failed.\n"); + module_put(pdev->dev.driver->owner); + return err; + } + } +#else /* CONFIG_OF */ + dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n"); + kbdev->arb.arb_dev = NULL; + kbdev->arb.arb_if = NULL; +#endif + return 0; +} + +void kbase_arbif_destroy(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); + } + kbdev->arb.arb_if = NULL; + if (kbdev->arb.arb_dev) + module_put(kbdev->arb.arb_dev->driver->owner); + kbdev->arb.arb_dev = NULL; +} + +void kbase_arbif_gpu_request(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_gpu_request(arb_if); + } +} + +void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { + dev_dbg(kbdev->dev, "%s\n", __func__); + KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED(kbdev, kbdev); + arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); + } +} + +void kbase_arbif_gpu_active(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_gpu_active(arb_if); + } +} + +void kbase_arbif_gpu_idle(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) { + dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n"); + arb_if->vm_ops.vm_arb_gpu_idle(arb_if); + } +} diff --git a/mali_kbase/arbiter/mali_kbase_arbif.h b/mali_kbase/arbiter/mali_kbase_arbif.h new file mode 100644 index 0000000..e7e9de7 --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbif.h @@ -0,0 +1,133 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Mali arbiter interface APIs to share GPU between Virtual Machines + */ + +#ifndef _MALI_KBASE_ARBIF_H_ +#define _MALI_KBASE_ARBIF_H_ + +/** + * enum kbase_arbif_evt - Internal Arbiter event. + * + * @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing + * and can be stopped + * @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter + * @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter + * @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter + * @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state. + * @KBASE_VM_REF_EVENT: KBase has transitioned into an active state. + * @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending + * @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming + */ +enum kbase_arbif_evt { + KBASE_VM_GPU_INITIALIZED_EVT = 1, + KBASE_VM_GPU_STOP_EVT, + KBASE_VM_GPU_GRANTED_EVT, + KBASE_VM_GPU_LOST_EVT, + KBASE_VM_GPU_IDLE_EVENT, + KBASE_VM_REF_EVENT, + KBASE_VM_OS_SUSPEND_EVENT, + KBASE_VM_OS_RESUME_EVENT, +}; + +/** + * kbase_arbif_init() - Initialize the arbiter interface functionality. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Initialize the arbiter interface and also determines + * if Arbiter functionality is required. + * + * Return: 0 if the Arbiter interface was successfully initialized or the + * Arbiter was not required. + */ +int kbase_arbif_init(struct kbase_device *kbdev); + +/** + * kbase_arbif_destroy() - Cleanups the arbiter interface functionality. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Cleans up the arbiter interface functionality and resets the reference count + * of the arbif module used + */ +void kbase_arbif_destroy(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_request() - Send GPU request message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to request GPU access. + */ +void kbase_arbif_gpu_request(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @gpu_required: true if GPU access is still required + * (Arbiter will automatically send another grant message) + * + * Sends a message to Arbiter to notify that the GPU has stopped. + * @note Once this call has been made, KBase must not attempt to access the GPU + * until the #KBASE_VM_GPU_GRANTED_EVT event has been received. + */ +void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required); + +/** + * kbase_arbif_gpu_active() - Send a GPU active message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to report that KBase has gone active. + */ +void kbase_arbif_gpu_active(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to report that KBase has gone idle. + */ +void kbase_arbif_gpu_idle(struct kbase_device *kbdev); + +#endif /* _MALI_KBASE_ARBIF_H_ */ diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_defs.h b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h new file mode 100644 index 0000000..1f53cbf --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h @@ -0,0 +1,95 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Mali structures define to support arbitration feature + */ + +#ifndef _MALI_KBASE_ARBITER_DEFS_H_ +#define _MALI_KBASE_ARBITER_DEFS_H_ + +#include "mali_kbase_arbiter_pm.h" + +/** + * struct kbase_arbiter_vm_state - Struct representing the state and containing the + * data of pm work + * @kbdev: Pointer to kbase device structure (must be a valid pointer) + * @vm_state_lock: The lock protecting the VM state when arbiter is used. + * This lock must also be held whenever the VM state is being + * transitioned + * @vm_state_wait: Wait queue set when GPU is granted + * @vm_state: Current state of VM + * @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use + * with the Arbiter + * @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU + * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU + * @vm_arb_starting: Work queue resume in progress + * @vm_arb_stopping: Work queue suspend in progress + * @vm_arb_users_waiting: Count of users waiting for GPU + */ +struct kbase_arbiter_vm_state { + struct kbase_device *kbdev; + struct mutex vm_state_lock; + wait_queue_head_t vm_state_wait; + enum kbase_vm_state vm_state; + struct workqueue_struct *vm_arb_wq; + struct work_struct vm_suspend_work; + struct work_struct vm_resume_work; + bool vm_arb_starting; + bool vm_arb_stopping; + int vm_arb_users_waiting; +}; + +/** + * struct kbase_arbiter_device - Representing an instance of arbiter device, + * allocated from the probe method of Mali driver + * @arb_if: Pointer to the arbiter interface device + * @arb_dev: Pointer to the arbiter device + */ +struct kbase_arbiter_device { + struct arbiter_if_dev *arb_if; + struct device *arb_dev; +}; + +#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */ diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_interface.h b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h new file mode 100644 index 0000000..5d5d8a7 --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h @@ -0,0 +1,181 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Defines the Mali arbiter interface + */ + +#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_ +#define _MALI_KBASE_ARBITER_INTERFACE_H_ + +/** + * @brief Mali arbiter interface version + * + * This specifies the current version of the configuration interface. Whenever + * the arbiter interface changes, so that integration effort is required, the + * version number will be increased. Each configuration must make an effort + * to check that it implements the correct version. + * + * Version history: + * 1 - Added the Mali arbiter configuration interface. + * 2 - Strip out reference code from header + * 3 - Removed DVFS utilization interface (DVFS moved to arbiter side) + */ +#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3 + +struct arbiter_if_dev; + +/** + * struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM + * + * This struct contains callbacks used to deliver messages + * from the arbiter to the corresponding VM. + * + * Note that calls into these callbacks may have synchronous calls back into + * the arbiter arbiter_if_vm_arb_ops callbacks below. + * For example vm_arb_gpu_stopped() may be called as a side effect of + * arb_vm_gpu_stop() being called here. + */ +struct arbiter_if_arb_vm_ops { + /** + * arb_vm_gpu_stop() - Ask VM to stop using GPU + * @dev: The arbif kernel module device. + * + * Informs KBase to stop using the GPU as soon as possible. + * @Note: Once the driver is no longer using the GPU, a call to + * vm_arb_gpu_stopped is expected by the arbiter. + */ + void (*arb_vm_gpu_stop)(struct device *dev); + + /** + * arb_vm_gpu_granted() - GPU has been granted to VM + * @dev: The arbif kernel module device. + * + * Informs KBase that the GPU can now be used by the VM. + */ + void (*arb_vm_gpu_granted)(struct device *dev); + + /** + * arb_vm_gpu_lost() - VM has lost the GPU + * @dev: The arbif kernel module device. + * + * This is called if KBase takes too long to respond to the arbiter + * stop request. + * Once this is called, KBase will assume that access to the GPU + * has been lost and will fail all running jobs and reset its + * internal state. + * If successful, will respond with a vm_arb_gpu_stopped message. + */ + void (*arb_vm_gpu_lost)(struct device *dev); +}; + +/** + * struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter + * + * This struct contains callbacks used to request operations + * from the VM to the arbiter + * + * Note that we must not make any synchronous calls back in to the VM + * (via arbiter_if_arb_vm_ops above) in the context of these callbacks. + */ +struct arbiter_if_vm_arb_ops { + /** + * vm_arb_register_dev() - Register VM device driver callbacks. + * @arbif_dev: The arbiter interface we are registering device callbacks + * @dev: The device structure to supply in the callbacks. + * @ops: The callbacks that the device driver supports + * (none are optional). + */ + int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev, + struct device *dev, struct arbiter_if_arb_vm_ops *ops); + + /** + * vm_arb_unregister_dev() - Unregister VM device driver callbacks. + * @arbif_dev: The arbiter interface we are unregistering from. + */ + void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_request() - Ask the arbiter interface for GPU access. + * @arbif_dev: The arbiter interface we want to issue the request. + */ + void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_active() - Inform arbiter that the driver has gone active + * @arbif_dev: The arbiter interface device. + */ + void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle + * @arbif_dev: The arbiter interface device. + */ + void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped + * using the GPU + * @arbif_dev: The arbiter interface device. + * @gpu_required: The GPU is still needed to do more work. + */ + void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev, + u8 gpu_required); +}; + +/** + * struct arbiter_if_dev - Arbiter Interface + * @vm_ops: Callback functions for connecting KBase with + * arbiter interface device. + * @priv_data: Internal arbif data not used by KBASE. + * + * Arbiter Interface Kernel Module State used for linking KBase + * with an arbiter interface platform device + */ +struct arbiter_if_dev { + struct arbiter_if_vm_arb_ops vm_ops; + void *priv_data; +}; + +#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */ diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.c b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c new file mode 100644 index 0000000..6c35e16 --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c @@ -0,0 +1,645 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_arbiter_pm.c + * Mali arbiter power manager state machine and APIs + */ + +#include <mali_kbase.h> +#include <mali_kbase_pm.h> +#include <mali_kbase_hwaccess_jm.h> +#include <mali_kbase_irq_internal.h> +#include <mali_kbase_hwcnt_context.h> +#include <mali_kbase_pm_internal.h> +#include <tl/mali_kbase_tracepoints.h> + +static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); +static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( + struct kbase_device *kbdev); + +static inline const char *kbase_arbiter_pm_vm_state_str( + enum kbase_vm_state state) +{ + switch (state) { + case KBASE_VM_STATE_INITIALIZING: + return "KBASE_VM_STATE_INITIALIZING"; + case KBASE_VM_STATE_INITIALIZING_WITH_GPU: + return "KBASE_VM_STATE_INITIALIZING_WITH_GPU"; + case KBASE_VM_STATE_SUSPENDED: + return "KBASE_VM_STATE_SUSPENDED"; + case KBASE_VM_STATE_STOPPED: + return "KBASE_VM_STATE_STOPPED"; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED"; + case KBASE_VM_STATE_STARTING: + return "KBASE_VM_STATE_STARTING"; + case KBASE_VM_STATE_IDLE: + return "KBASE_VM_STATE_IDLE"; + case KBASE_VM_STATE_ACTIVE: + return "KBASE_VM_STATE_ACTIVE"; + case KBASE_VM_STATE_STOPPING_IDLE: + return "KBASE_VM_STATE_STOPPING_IDLE"; + case KBASE_VM_STATE_STOPPING_ACTIVE: + return "KBASE_VM_STATE_STOPPING_ACTIVE"; + case KBASE_VM_STATE_SUSPEND_PENDING: + return "KBASE_VM_STATE_SUSPEND_PENDING"; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT"; + default: + KBASE_DEBUG_ASSERT(false); + return "[UnknownState]"; + } +} + +static inline const char *kbase_arbiter_pm_vm_event_str( + enum kbase_arbif_evt evt) +{ + switch (evt) { + case KBASE_VM_GPU_INITIALIZED_EVT: + return "KBASE_VM_GPU_INITIALIZED_EVT"; + case KBASE_VM_GPU_STOP_EVT: + return "KBASE_VM_GPU_STOP_EVT"; + case KBASE_VM_GPU_GRANTED_EVT: + return "KBASE_VM_GPU_GRANTED_EVT"; + case KBASE_VM_GPU_LOST_EVT: + return "KBASE_VM_GPU_LOST_EVT"; + case KBASE_VM_OS_SUSPEND_EVENT: + return "KBASE_VM_OS_SUSPEND_EVENT"; + case KBASE_VM_OS_RESUME_EVENT: + return "KBASE_VM_OS_RESUME_EVENT"; + case KBASE_VM_GPU_IDLE_EVENT: + return "KBASE_VM_GPU_IDLE_EVENT"; + case KBASE_VM_REF_EVENT: + return "KBASE_VM_REF_EVENT"; + default: + KBASE_DEBUG_ASSERT(false); + return "[UnknownEvent]"; + } +} + +static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, + enum kbase_vm_state new_state) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + dev_dbg(kbdev->dev, "VM set_state %s -> %s", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state), + kbase_arbiter_pm_vm_state_str(new_state)); + lockdep_assert_held(&arb_vm_state->vm_state_lock); + arb_vm_state->vm_state = new_state; + wake_up(&arb_vm_state->vm_state_wait); +} + +static void kbase_arbiter_pm_suspend_wq(struct work_struct *data) +{ + struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, + struct kbase_arbiter_vm_state, + vm_suspend_work); + struct kbase_device *kbdev = arb_vm_state->kbdev; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">%s\n", __func__); + if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE || + arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPING_ACTIVE || + arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPEND_PENDING) { + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n"); + kbase_pm_driver_suspend(kbdev); + dev_dbg(kbdev->dev, "<kbase_pm_driver_suspend\n"); + mutex_lock(&arb_vm_state->vm_state_lock); + } + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "<%s\n", __func__); +} + +static void kbase_arbiter_pm_resume_wq(struct work_struct *data) +{ + struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, + struct kbase_arbiter_vm_state, + vm_resume_work); + struct kbase_device *kbdev = arb_vm_state->kbdev; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">%s\n", __func__); + arb_vm_state->vm_arb_starting = true; + if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) { + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n"); + kbase_pm_driver_resume(kbdev, true); + dev_dbg(kbdev->dev, "<kbase_pm_driver_resume\n"); + mutex_lock(&arb_vm_state->vm_state_lock); + } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { + kbase_arbiter_pm_vm_stopped(kbdev); + } + arb_vm_state->vm_arb_starting = false; + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "<%s\n", __func__); +} + +int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) +{ + int err; + struct kbase_arbiter_vm_state *arb_vm_state = NULL; + + arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state), + GFP_KERNEL); + if (arb_vm_state == NULL) + return -ENOMEM; + + arb_vm_state->kbdev = kbdev; + arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING; + + mutex_init(&arb_vm_state->vm_state_lock); + init_waitqueue_head(&arb_vm_state->vm_state_wait); + arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq", + WQ_HIGHPRI); + if (!arb_vm_state->vm_arb_wq) { + dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n"); + return -ENOMEM; + } + INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq); + INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq); + arb_vm_state->vm_arb_starting = false; + arb_vm_state->vm_arb_users_waiting = 0; + kbdev->pm.arb_vm_state = arb_vm_state; + + err = kbase_arbif_init(kbdev); + if (err) { + goto arbif_init_fail; + } + if (kbdev->arb.arb_if) { + kbase_arbif_gpu_request(kbdev); + dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state == + KBASE_VM_STATE_INITIALIZING_WITH_GPU); + dev_dbg(kbdev->dev, + "Waiting for initial GPU assignment - done\n"); + } + return 0; + +arbif_init_fail: + destroy_workqueue(arb_vm_state->vm_arb_wq); + kfree(arb_vm_state); + kbdev->pm.arb_vm_state = NULL; + return err; +} + +void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + mutex_lock(&arb_vm_state->vm_state_lock); + if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) + kbase_arbif_gpu_stopped(kbdev, false); + + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbif_destroy(kbdev); + destroy_workqueue(arb_vm_state->vm_arb_wq); + arb_vm_state->vm_arb_wq = NULL; + kfree(kbdev->pm.arb_vm_state); + kbdev->pm.arb_vm_state = NULL; +} + +void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + mutex_lock(&arb_vm_state->vm_state_lock); + if (!kbdev->arb.arb_if || + arb_vm_state->vm_state > + KBASE_VM_STATE_STOPPED_GPU_REQUESTED) + kbase_release_interrupts(kbdev); + + mutex_unlock(&arb_vm_state->vm_state_lock); +} + +void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) +{ + bool request_gpu = false; + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + + if (arb_vm_state->vm_arb_users_waiting > 0 && + arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + + dev_dbg(kbdev->dev, "%s %s\n", __func__, + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + kbase_release_interrupts(kbdev); + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPING_ACTIVE: + request_gpu = true; + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + break; + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED); + break; + case KBASE_VM_STATE_SUSPEND_PENDING: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + dev_warn(kbdev->dev, "unexpected pm_stop VM state %u", + arb_vm_state->vm_state); + break; + } + + kbase_arbif_gpu_stopped(kbdev, request_gpu); +} + +static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_INITIALIZING: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_INITIALIZING_WITH_GPU); + break; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); + kbase_install_interrupts(kbdev); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_resume_work); + break; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + kbase_arbif_gpu_stopped(kbdev, false); + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + dev_warn(kbdev->dev, + "GPU_GRANTED when not expected - state %s\n", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + break; + } +} + +static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) { + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + } + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_IDLE); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_STARTING: + dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING."); + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + if (arb_vm_state->vm_arb_starting) + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_SUSPEND_PENDING: + /* Suspend finishes with a stop so nothing else to do */ + break; + default: + dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + break; + } +} + +static void kbase_gpu_lost(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STARTING: + case KBASE_VM_STATE_ACTIVE: + case KBASE_VM_STATE_IDLE: + dev_warn(kbdev->dev, "GPU lost in state %s", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + kbase_arbiter_pm_vm_gpu_stop(kbdev); + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_pm_handle_gpu_lost(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + break; + case KBASE_VM_STATE_STOPPING_IDLE: + case KBASE_VM_STATE_STOPPING_ACTIVE: + case KBASE_VM_STATE_SUSPEND_PENDING: + dev_info(kbdev->dev, "GPU lost while stopping"); + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_pm_handle_gpu_lost(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + break; + case KBASE_VM_STATE_SUSPENDED: + case KBASE_VM_STATE_STOPPED: + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + dev_info(kbdev->dev, "GPU lost while already stopped"); + break; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + dev_info(kbdev->dev, "GPU lost while waiting to suspend"); + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + break; + } + +} + +static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state( + struct kbase_device *kbdev) +{ + switch (kbdev->pm.arb_vm_state->vm_state) { + case KBASE_VM_STATE_SUSPENDED: + case KBASE_VM_STATE_STOPPED: + case KBASE_VM_STATE_IDLE: + case KBASE_VM_STATE_ACTIVE: + return true; + default: + return false; + } +} + +static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + enum kbase_vm_state prev_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + if (kbdev->arb.arb_if) { + if (kbdev->pm.arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPENDED) + return; + } + /* Block suspend OS function until we are in a stable state + * with vm_state_lock + */ + while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) { + prev_state = arb_vm_state->vm_state; + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPING_ACTIVE: + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + break; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT); + break; + case KBASE_VM_STATE_STARTING: + if (!arb_vm_state->vm_arb_starting) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + kbase_arbiter_pm_vm_stopped(kbdev); + } + break; + default: + break; + } + mutex_unlock(&arb_vm_state->vm_state_lock); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state != prev_state); + mutex_lock(&arb_vm_state->vm_state_lock); + } + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPED: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPENDED); + break; + case KBASE_VM_STATE_IDLE: + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + mutex_unlock(&arb_vm_state->vm_state_lock); + /* Ensure resume has completed fully before starting suspend */ + flush_work(&arb_vm_state->vm_resume_work); + kbase_pm_driver_suspend(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + break; + case KBASE_VM_STATE_SUSPENDED: + break; + default: + KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend"); + break; + } +} + +static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPENDED, + "Unexpected state to resume"); + + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + + /* Release lock and block resume OS function until we have + * asynchronously received the GRANT message from the Arbiter and + * fully resumed + */ + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + flush_work(&arb_vm_state->vm_resume_work); + mutex_lock(&arb_vm_state->vm_state_lock); +} + +void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, + enum kbase_arbif_evt evt) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + if (!kbdev->arb.arb_if) + return; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "%s %s\n", __func__, + kbase_arbiter_pm_vm_event_str(evt)); + + switch (evt) { + case KBASE_VM_GPU_GRANTED_EVT: + kbase_arbiter_pm_vm_gpu_start(kbdev); + break; + case KBASE_VM_GPU_STOP_EVT: + kbase_arbiter_pm_vm_gpu_stop(kbdev); + break; + case KBASE_VM_GPU_LOST_EVT: + dev_info(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); + kbase_gpu_lost(kbdev); + break; + case KBASE_VM_OS_SUSPEND_EVENT: + kbase_arbiter_pm_vm_os_prepare_suspend(kbdev); + break; + case KBASE_VM_OS_RESUME_EVENT: + kbase_arbiter_pm_vm_os_resume(kbdev); + break; + case KBASE_VM_GPU_IDLE_EVENT: + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_IDLE); + kbase_arbif_gpu_idle(kbdev); + break; + default: + break; + } + break; + + case KBASE_VM_REF_EVENT: + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STARTING: + KBASE_TLSTREAM_TL_EVENT_ARB_STARTED(kbdev, kbdev); + /* FALL THROUGH */ + case KBASE_VM_STATE_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_ACTIVE); + kbase_arbif_gpu_active(kbdev); + break; + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + break; + default: + break; + } + break; + + case KBASE_VM_GPU_INITIALIZED_EVT: + lockdep_assert_held(&kbdev->pm.lock); + if (kbdev->pm.active_count > 0) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_ACTIVE); + kbase_arbif_gpu_active(kbdev); + } else { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_IDLE); + kbase_arbif_gpu_idle(kbdev); + } + break; + + default: + dev_alert(kbdev->dev, "Got Unknown Event!"); + break; + } + mutex_unlock(&arb_vm_state->vm_state_lock); +} + +static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n"); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || + arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); + dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n"); +} + +static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( + struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || + arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); +} + +int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + if (kbdev->arb.arb_if) { + mutex_lock(&arb_vm_state->vm_state_lock); + while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) { + /* Update VM state since we have GPU work to do */ + if (arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + else if (arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPED) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + } else if (arb_vm_state->vm_state == + KBASE_VM_STATE_INITIALIZING_WITH_GPU) + break; + + if (suspend_handler != + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { + if (suspend_handler == + KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED + || + kbdev->pm.active_count > 0) + break; + + mutex_unlock(&arb_vm_state->vm_state_lock); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + return 1; + } + + if (arb_vm_state->vm_state == + KBASE_VM_STATE_INITIALIZING_WITH_GPU) + break; + + /* Need to synchronously wait for GPU assignment */ + arb_vm_state->vm_arb_users_waiting++; + mutex_unlock(&arb_vm_state->vm_state_lock); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + mutex_lock(&arb_vm_state->vm_state_lock); + arb_vm_state->vm_arb_users_waiting--; + } + mutex_unlock(&arb_vm_state->vm_state_lock); + } + return 0; +} diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.h b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h new file mode 100644 index 0000000..3c49eb1 --- /dev/null +++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h @@ -0,0 +1,159 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file + * Mali arbiter power manager state machine and APIs + */ + +#ifndef _MALI_KBASE_ARBITER_PM_H_ +#define _MALI_KBASE_ARBITER_PM_H_ + +#include "mali_kbase_arbif.h" + +/** + * enum kbase_vm_state - Current PM Arbitration state. + * + * @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized. + * @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU + * has been granted. + * @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned. + * @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required. + * @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase + * but a request has been made. + * @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run. + * @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do + * @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it + * @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode. + * @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU + * has already been requested. + * In this situation we must wait for + * the Arbiter to send a GRANTED message + * and respond immediately with + * a STOPPED message before entering + * the suspend mode. + * @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there + * is currently no work to do on the GPU. + * @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when + * KBase has work to do. + */ +enum kbase_vm_state { + KBASE_VM_STATE_INITIALIZING, + KBASE_VM_STATE_INITIALIZING_WITH_GPU, + KBASE_VM_STATE_SUSPENDED, + KBASE_VM_STATE_STOPPED, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED, + KBASE_VM_STATE_STARTING, + KBASE_VM_STATE_IDLE, + KBASE_VM_STATE_ACTIVE, + KBASE_VM_STATE_SUSPEND_PENDING, + KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT, + KBASE_VM_STATE_STOPPING_IDLE, + KBASE_VM_STATE_STOPPING_ACTIVE +}; + +/** + * kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Initialize the arbiter and other required resources during the runtime + * and request the GPU for the VM for the first time. + * + * Return: 0 if successful, otherwise a standard Linux error code + */ +int kbase_arbiter_pm_early_init(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Clean up all the resources + */ +void kbase_arbiter_pm_early_term(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Releases interrupts if needed (GPU is available) otherwise does nothing + */ +void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * The state machine function. Receives events and transitions states + * according the event received and the current state + */ +void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, + enum kbase_arbif_evt event); + +/** + * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for + * arbitration mode + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @suspend_handler: The handler code for how to handle a suspend + * that might occur + * + * This function handles a suspend event from the driver, + * communicating with the arbiter and waiting synchronously for the GPU + * to be granted again depending on the VM state. + * + * Return: 0 if success, 1 if failure due to system suspending/suspended + */ +int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler); + + +/** + * kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This function handles a stop event for the VM. + * It will update the VM state and forward the stop event to the driver. + */ +void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev); + +#endif /*_MALI_KBASE_ARBITER_PM_H_ */ diff --git a/mali_kbase/backend/gpu/mali_kbase_device_hw.c b/mali_kbase/backend/gpu/mali_kbase_device_hw.c index 0ec8cef..8bdc526 100644 --- a/mali_kbase/backend/gpu/mali_kbase_device_hw.c +++ b/mali_kbase/backend/gpu/mali_kbase_device_hw.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -190,6 +190,15 @@ u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset) } KBASE_EXPORT_TEST_API(kbase_reg_read); + +bool kbase_is_gpu_lost(struct kbase_device *kbdev) +{ + u32 val; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + + return val == 0; +} #endif /* !defined(CONFIG_MALI_NO_MALI) */ /** @@ -245,7 +254,7 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev) kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | CLEAN_CACHES_COMPLETED); - KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES); @@ -280,7 +289,7 @@ static void kbase_clean_caches_done(struct kbase_device *kbdev) if (kbdev->cache_clean_queued) { kbdev->cache_clean_queued = false; - KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES); } else { @@ -333,7 +342,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) { bool clear_gpu_fault = false; - KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); if (val & GPU_FAULT) clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev, val & MULTIPLE_GPU_FAULTS); @@ -344,7 +353,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) if (val & PRFCNT_SAMPLE_COMPLETED) kbase_instr_hwcnt_sample_done(kbdev); - KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must @@ -374,5 +383,5 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) } - KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); } diff --git a/mali_kbase/backend/gpu/mali_kbase_device_internal.h b/mali_kbase/backend/gpu/mali_kbase_device_internal.h index 5ddc4a5..2e1d011 100644 --- a/mali_kbase/backend/gpu/mali_kbase_device_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_device_internal.h @@ -51,6 +51,18 @@ void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value); u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset); /** + * kbase_is_gpu_lost() - Has the GPU been lost. + * @kbdev: Kbase device pointer + * + * This function will return true if the GPU has been lost. + * When this happens register reads will be zero. A zero GPU_ID is + * invalid so this is used to detect GPU_LOST + * + * Return: True if GPU LOST + */ +bool kbase_is_gpu_lost(struct kbase_device *kbdev); + +/** * kbase_gpu_start_cache_clean - Start a cache clean * @kbdev: Kbase device * diff --git a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c index ae3ed9c..352afa1 100644 --- a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c @@ -1,6 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,8 +21,6 @@ * */ - - /* * Base kernel property query backend APIs */ @@ -31,79 +30,93 @@ #include <backend/gpu/mali_kbase_pm_internal.h> #include <mali_kbase_hwaccess_gpuprops.h> -void kbase_backend_gpuprops_get(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump) { int i; + struct kbase_gpuprops_regdump registers; /* Fill regdump with the content of the relevant registers */ - regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); - regdump->l2_features = kbase_reg_read(kbdev, + registers.l2_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES)); - regdump->core_features = kbase_reg_read(kbdev, + registers.core_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(CORE_FEATURES)); - regdump->tiler_features = kbase_reg_read(kbdev, + registers.tiler_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_FEATURES)); - regdump->mem_features = kbase_reg_read(kbdev, + registers.mem_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(MEM_FEATURES)); - regdump->mmu_features = kbase_reg_read(kbdev, + registers.mmu_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(MMU_FEATURES)); - regdump->as_present = kbase_reg_read(kbdev, + registers.as_present = kbase_reg_read(kbdev, GPU_CONTROL_REG(AS_PRESENT)); - regdump->js_present = kbase_reg_read(kbdev, + registers.js_present = kbase_reg_read(kbdev, GPU_CONTROL_REG(JS_PRESENT)); for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) - regdump->js_features[i] = kbase_reg_read(kbdev, + registers.js_features[i] = kbase_reg_read(kbdev, GPU_CONTROL_REG(JS_FEATURES_REG(i))); for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) - regdump->texture_features[i] = kbase_reg_read(kbdev, + registers.texture_features[i] = kbase_reg_read(kbdev, GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i))); - regdump->thread_max_threads = kbase_reg_read(kbdev, + registers.thread_max_threads = kbase_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_THREADS)); - regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, + registers.thread_max_workgroup_size = kbase_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE)); - regdump->thread_max_barrier_size = kbase_reg_read(kbdev, + registers.thread_max_barrier_size = kbase_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE)); - regdump->thread_features = kbase_reg_read(kbdev, + registers.thread_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(THREAD_FEATURES)); - regdump->thread_tls_alloc = kbase_reg_read(kbdev, + registers.thread_tls_alloc = kbase_reg_read(kbdev, GPU_CONTROL_REG(THREAD_TLS_ALLOC)); - regdump->shader_present_lo = kbase_reg_read(kbdev, + registers.shader_present_lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_LO)); - regdump->shader_present_hi = kbase_reg_read(kbdev, + registers.shader_present_hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_HI)); - regdump->tiler_present_lo = kbase_reg_read(kbdev, + registers.tiler_present_lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_LO)); - regdump->tiler_present_hi = kbase_reg_read(kbdev, + registers.tiler_present_hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_HI)); - regdump->l2_present_lo = kbase_reg_read(kbdev, + registers.l2_present_lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_LO)); - regdump->l2_present_hi = kbase_reg_read(kbdev, + registers.l2_present_hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_HI)); - regdump->stack_present_lo = kbase_reg_read(kbdev, + registers.stack_present_lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(STACK_PRESENT_LO)); - regdump->stack_present_hi = kbase_reg_read(kbdev, + registers.stack_present_hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(STACK_PRESENT_HI)); + + if (!kbase_is_gpu_lost(kbdev)) { + *regdump = registers; + return 0; + } else + return -EIO; } -void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump) { if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { + u32 coherency_features; + /* Ensure we can access the GPU registers */ kbase_pm_register_access_enable(kbdev); - regdump->coherency_features = kbase_reg_read(kbdev, + coherency_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(COHERENCY_FEATURES)); + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + + regdump->coherency_features = coherency_features; + /* We're done accessing the GPU registers for now. */ kbase_pm_register_access_disable(kbdev); } else { @@ -112,13 +125,22 @@ void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, COHERENCY_FEATURE_BIT(COHERENCY_NONE) | COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); } + + return 0; } -void kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump) { if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { - regdump->l2_features = kbase_reg_read(kbdev, + u32 l2_features = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES)); + + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + + regdump->l2_features = l2_features; } + + return 0; } diff --git a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c index cb3e1d3..8b320c7 100644 --- a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c @@ -212,8 +212,8 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) kbdev->hwcnt.addr >> 32); /* Start dumping */ - KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, - kbdev->hwcnt.addr, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, + kbdev->hwcnt.addr); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE); @@ -359,7 +359,7 @@ int kbase_instr_hwcnt_clear(struct kbase_context *kctx) goto out; /* Clear the counters */ - KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR); diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_as.c b/mali_kbase/backend/gpu/mali_kbase_jm_as.c index 5b30e93..bb4f548 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_as.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_as.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -148,8 +148,7 @@ int kbase_backend_find_and_release_free_address_space( */ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && atomic_read(&as_kctx->refcount) == 1) { - if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, - as_kctx)) { + if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) { WARN(1, "Failed to retain active context\n"); spin_unlock_irqrestore(&kbdev->hwaccess_lock, @@ -236,7 +235,7 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev, if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { /* We need to retain it to keep the corresponding address space */ - kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + kbase_ctx_sched_retain_ctx_refcount(kctx); } return true; diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c index 819edaf..fa6bc83 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c @@ -28,6 +28,7 @@ #include <mali_kbase_config.h> #include <gpu/mali_kbase_gpu_regmap.h> #include <tl/mali_kbase_tracepoints.h> +#include <mali_linux_trace.h> #include <mali_kbase_hw.h> #include <mali_kbase_hwaccess_jm.h> #include <mali_kbase_reset_gpu.h> @@ -39,12 +40,6 @@ static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); -static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, - struct kbase_context *kctx) -{ - return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)); -} - static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req core_req, int js) @@ -129,6 +124,10 @@ static u64 select_job_chain(struct kbase_jd_atom *katom) if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) return jc; + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + rp = &kctx->jctx.renderpasses[katom->renderpass_id]; /* We can read a subset of renderpass state without holding * higher-level locks (but not end_katom, for example). @@ -259,7 +258,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx", katom, kctx, js, jc_head); - KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, + KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, (u32)affinity); KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx, @@ -291,6 +290,10 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; } #endif + + trace_sysgraph_gpu(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom), js); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_START); } @@ -358,7 +361,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) KBASE_DEBUG_ASSERT(kbdev); - KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); + KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done); end_timestamp = ktime_get(); @@ -543,7 +546,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) */ kbasep_try_reset_gpu_early_locked(kbdev); } - KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); + KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count); } void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, @@ -552,7 +555,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) { -#if KBASE_TRACE_ENABLE +#if KBASE_KTRACE_ENABLE u32 status_reg_before; u64 job_in_head_before; u32 status_reg_after; @@ -608,7 +611,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action); -#if KBASE_TRACE_ENABLE +#if KBASE_KTRACE_ENABLE status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS)); if (status_reg_after == BASE_JD_EVENT_ACTIVE) { struct kbase_jd_atom *head; @@ -618,36 +621,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, head_kctx = head->kctx; if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, - head, job_in_head_before, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js); else - KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); switch (action) { case JS_COMMAND_SOFT_STOP: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js); break; case JS_COMMAND_SOFT_STOP_0: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js); break; case JS_COMMAND_SOFT_STOP_1: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js); break; case JS_COMMAND_HARD_STOP: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js); break; case JS_COMMAND_HARD_STOP_0: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js); break; case JS_COMMAND_HARD_STOP_1: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, - head, head->jc, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js); break; default: BUG(); @@ -655,36 +650,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, } } else { if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, - job_in_head_before, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js); else - KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); switch (action) { case JS_COMMAND_SOFT_STOP: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, - js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js); break; case JS_COMMAND_SOFT_STOP_0: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js); break; case JS_COMMAND_SOFT_STOP_1: - KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js); break; case JS_COMMAND_HARD_STOP: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, - js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js); break; case JS_COMMAND_HARD_STOP_0: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js); break; case JS_COMMAND_HARD_STOP_1: - KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, - 0, js); + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js); break; default: BUG(); @@ -766,9 +753,9 @@ static int softstop_start_rp_nolock( return -EPERM; } - if (WARN_ON(katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return -EINVAL; + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[katom->renderpass_id]; if (WARN_ON(rp->state != KBASE_JD_RP_START && @@ -1058,7 +1045,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) KBASE_RESET_GPU_SILENT) silent = true; - KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); /* Disable GPU hardware counters. * This call will block until counters are disabled. @@ -1186,7 +1173,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) dev_err(kbdev->dev, "Reset complete"); /* Try submitting some jobs to restart processing */ - KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); kbase_js_sched_all(kbdev); /* Process any pending slot updates */ @@ -1201,7 +1188,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); } static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h index d1ed42d..1419b59 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h @@ -34,6 +34,7 @@ #include <linux/atomic.h> #include <backend/gpu/mali_kbase_jm_rb.h> +#include <backend/gpu/mali_kbase_device_internal.h> /** * kbase_job_submit_nolock() - Submit a job to a certain job-slot @@ -70,6 +71,13 @@ static inline char *kbasep_make_job_slot_string(int js, char *js_string, } #endif +static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, + struct kbase_context *kctx) +{ + return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)); +} + + /** * kbase_job_hw_submit() - Submit a job to the GPU * @kbdev: Device pointer diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c index 6daea01..ec7bcb1 100644 --- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c +++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c @@ -614,11 +614,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, return -EAGAIN; } - if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || - kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { + if (kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2) || + kbase_is_gpu_lost(kbdev)) { /* - * The L2 is still powered, wait for all the users to - * finish with it before doing the actual reset. + * The L2 is still powered, wait for all + * the users to finish with it before doing + * the actual reset. */ return -EAGAIN; } @@ -806,7 +810,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_lost(kbdev)) +#else if (kbase_reset_gpu_is_active(kbdev)) +#endif return; for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { @@ -957,11 +965,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) other_slots_busy(kbdev, js)) break; - if ((kbdev->serialize_jobs & - KBASE_SERIALIZE_RESET) && - kbase_reset_gpu_is_active(kbdev)) +#ifdef CONFIG_MALI_GEM5_BUILD + if (!kbasep_jm_is_js_free(kbdev, js, + katom[idx]->kctx)) break; - +#endif /* Check if this job needs the cycle counter * enabled before submission */ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) @@ -1124,8 +1132,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, kbase_gpu_exception_name( completion_code)); -#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 - KBASE_TRACE_DUMP(kbdev); +#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0 + KBASE_KTRACE_DUMP(kbdev); #endif kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); @@ -1179,8 +1187,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, } } - KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, - js, completion_code); + KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code); if (job_tail != 0 && job_tail != katom->jc) { /* Some of the job has been executed */ @@ -1189,7 +1196,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, (void *)katom, job_tail); katom->jc = job_tail; - KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); } diff --git a/mali_kbase/backend/gpu/mali_kbase_js_backend.c b/mali_kbase/backend/gpu/mali_kbase_js_backend.c index c4df191..fcc0437 100644 --- a/mali_kbase/backend/gpu/mali_kbase_js_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_js_backend.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -301,8 +301,7 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), HRTIMER_MODE_REL); - KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, - 0u); + KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); } } diff --git a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c index 916916d..e67d12b 100644 --- a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c +++ b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -21,7 +21,6 @@ * */ - #include <mali_kbase.h> #include <mali_kbase_bits.h> #include <mali_kbase_config_defaults.h> @@ -76,7 +75,7 @@ static const struct l2_mmu_config_limit limits[] = { {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, }; -void kbase_set_mmu_quirks(struct kbase_device *kbdev) +int kbase_set_mmu_quirks(struct kbase_device *kbdev) { /* All older GPUs had 2 bits for both fields, this is a default */ struct l2_mmu_config_limit limit = { @@ -101,18 +100,23 @@ void kbase_set_mmu_quirks(struct kbase_device *kbdev) mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)); + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + mmu_config &= ~(limit.read.mask | limit.write.mask); /* Can't use FIELD_PREP() macro here as the mask isn't constant */ mmu_config |= (limit.read.value << limit.read.shift) | - (limit.write.value << limit.write.shift); + (limit.write.value << limit.write.shift); kbdev->hw_quirks_mmu = mmu_config; if (kbdev->system_coherency == COHERENCY_ACE) { /* Allow memory configuration disparity to be ignored, - * we optimize the use of shared memory and thus we - * expect some disparity in the memory configuration. - */ + * we optimize the use of shared memory and thus we + * expect some disparity in the memory configuration. + */ kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; } + + return 0; } diff --git a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h index 25636ee..0c779ac 100644 --- a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h +++ b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h @@ -20,13 +20,22 @@ * *//* SPDX-License-Identifier: GPL-2.0 */ /* - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms * of such GNU licence. * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * */ #ifndef _KBASE_L2_MMU_CONFIG_H_ @@ -38,7 +47,9 @@ * * Use this function to initialise the hw_quirks_mmu field, for instance to set * the MAX_READS and MAX_WRITES to sane defaults for each GPU. + * + * Return: Zero for succeess or a Linux error code */ -void kbase_set_mmu_quirks(struct kbase_device *kbdev); +int kbase_set_mmu_quirks(struct kbase_device *kbdev); #endif /* _KBASE_L2_MMU_CONFIG_H */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c index 6a9cb13..576c9f2 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c @@ -37,6 +37,7 @@ #include <backend/gpu/mali_kbase_pm_internal.h> #include <backend/gpu/mali_kbase_devfreq.h> #include <mali_kbase_dummy_job_wa.h> +#include <mali_kbase_irq_internal.h> static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); static void kbase_pm_hwcnt_disable_worker(struct work_struct *data); @@ -139,6 +140,9 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) kbdev->pm.backend.ca_cores_enabled = ~0ull; kbdev->pm.backend.gpu_powered = false; kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbdev->pm.gpu_lost = false; +#endif #ifdef CONFIG_MALI_DEBUG kbdev->pm.backend.driver_ready_for_irqs = false; #endif /* CONFIG_MALI_DEBUG */ @@ -245,7 +249,6 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) pm.backend.gpu_poweroff_wait_work); struct kbase_pm_device_data *pm = &kbdev->pm; struct kbase_pm_backend_data *backend = &pm->backend; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; unsigned long flags; /* Wait for power transitions to complete. We do this with no locks held @@ -253,8 +256,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) */ kbase_pm_wait_for_desired_state(kbdev); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); if (!backend->poweron_required) { unsigned long flags; @@ -272,11 +274,9 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) * process. Interrupts are disabled so no more faults * should be generated at this point. */ - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); kbase_flush_mmu_wqs(kbdev); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); /* Turn off clock now that fault have been handled. We * dropped locks so poweron_required may have changed - @@ -302,8 +302,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); wake_up(&kbdev->pm.backend.poweroff_wait); } @@ -518,14 +517,12 @@ KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete); int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; unsigned long irq_flags; int ret; KBASE_DEBUG_ASSERT(kbdev != NULL); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); /* A suspend won't happen during startup/insmod */ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); @@ -534,8 +531,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, * them. */ ret = kbase_pm_init_hw(kbdev, flags); if (ret) { - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); return ret; } @@ -565,8 +561,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, /* Turn on the GPU and any cores needed by the policy */ kbase_pm_do_poweron(kbdev, false); - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); return 0; } @@ -654,20 +649,16 @@ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - /* Force power off the GPU and all cores (regardless of policy), only * after the PM active count reaches zero (otherwise, we risk turning it * off prematurely) */ - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); kbase_pm_do_poweroff(kbdev); kbase_backend_timer_suspend(kbdev); - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); kbase_pm_wait_for_poweroff_complete(kbdev); @@ -677,16 +668,80 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbdev->pm.gpu_lost = false; +#endif kbase_pm_do_poweron(kbdev, true); kbase_backend_timer_resume(kbdev); + kbase_pm_unlock(kbdev); +} + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) +{ + unsigned long flags; + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + ktime_t end_timestamp = ktime_get(); + + /* Full GPU reset will have been done by hypervisor, so cancel */ + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + + /* GPU is no longer mapped to VM. So no interrupts will be received + * and Mali registers have been replaced by dummy RAM + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + spin_lock(&kbdev->mmu_mask_change); + kbdev->irq_reset_flush = true; + spin_unlock(&kbdev->mmu_mask_change); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_synchronize_irqs(kbdev); + kbase_flush_mmu_wqs(kbdev); + kbdev->irq_reset_flush = false; + + /* Clear all jobs running on the GPU */ + mutex_lock(&kbdev->pm.lock); + kbdev->pm.gpu_lost = true; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + if (!kbdev->pm.backend.protected_entry_transition_override) + kbase_backend_reset(kbdev, &end_timestamp); + kbase_pm_metrics_update(kbdev, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Cancel any pending HWC dumps */ + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Wait for all threads keeping GPU active to complete */ + mutex_unlock(&kbdev->pm.lock); + wait_event(kbdev->pm.zero_active_count_wait, + kbdev->pm.active_count == 0); + mutex_lock(&kbdev->pm.lock); + + /* Update state to GPU off */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.shaders_desired = false; + kbdev->pm.backend.l2_desired = false; + backend->l2_state = KBASE_L2_OFF; + backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; + kbdev->pm.backend.gpu_powered = false; + backend->poweroff_wait_in_progress = false; + KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); + wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); + kbase_gpu_cache_clean_wait_complete(kbdev); + backend->poweroff_wait_in_progress = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + wake_up(&kbdev->pm.backend.poweroff_wait); mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c index b691524..e7eef26 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c @@ -66,7 +66,7 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) } if (kbase_dummy_job_wa_enabled(kbdev)) { - dev_info_once(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled"); + dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled"); goto unlock; } diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c index b04d705..6b821f7 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -20,8 +21,6 @@ * */ - - /* * Base kernel Power Management hardware control */ @@ -43,6 +42,9 @@ #include <backend/gpu/mali_kbase_pm_internal.h> #include <backend/gpu/mali_kbase_l2_mmu_config.h> #include <mali_kbase_dummy_job_wa.h> +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include <arbiter/mali_kbase_arbiter_pm.h> +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #include <linux/of.h> @@ -281,16 +283,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, if (action == ACTION_PWRON) switch (core_type) { case KBASE_PM_CORE_SHADER: - KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, - lo); + KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores); break; case KBASE_PM_CORE_TILER: - KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, - NULL, 0u, lo); + KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores); break; case KBASE_PM_CORE_L2: - KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, - 0u, lo); + KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores); break; default: break; @@ -298,16 +297,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, else if (action == ACTION_PWROFF) switch (core_type) { case KBASE_PM_CORE_SHADER: - KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, - 0u, lo); + KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores); break; case KBASE_PM_CORE_TILER: - KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, - NULL, 0u, lo); + KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores); break; case KBASE_PM_CORE_L2: - KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, - 0u, lo); + KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores); /* disable snoops before L2 is turned off */ kbase_pm_cache_snoop_disable(kbdev); break; @@ -444,16 +440,13 @@ u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, switch (type) { case KBASE_PM_CORE_SHADER: - KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, - (u32) result); + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result); break; case KBASE_PM_CORE_TILER: - KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, - (u32) result); + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result); break; case KBASE_PM_CORE_L2: - KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, - (u32) result); + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result); break; default: break; @@ -543,7 +536,7 @@ static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) return strings[state]; } -static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) +static int kbase_pm_l2_update_state(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present; @@ -563,6 +556,13 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) u64 tiler_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); + /* + * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores + * are vulnerable to corruption if gpu is lost + */ + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + /* mask off ready from trans in case transitions finished * between the register reads */ @@ -600,10 +600,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_PEND_ON: if (!l2_trans && l2_ready == l2_present && !tiler_trans && tiler_ready == tiler_present) { - KBASE_TRACE_ADD(kbdev, - PM_CORES_CHANGE_AVAILABLE_TILER, - NULL, NULL, 0u, - (u32)tiler_ready); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready); /* * Ensure snoops are enabled after L2 is powered * up. Note that kbase keeps track of the snoop @@ -773,8 +770,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) kbase_gpu_start_cache_clean_nolock( kbdev); - KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, - NULL, NULL, 0u, 0u); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); backend->l2_state = KBASE_L2_PEND_OFF; break; @@ -819,8 +815,6 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) &kbdev->pm.backend.gpu_poweroff_wait_work); } - if (backend->l2_state == KBASE_L2_ON) - return l2_present; return 0; } @@ -897,13 +891,14 @@ static const char *kbase_shader_core_state_to_string( return strings[state]; } -static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) +static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer; enum kbase_shader_core_state prev_state; u64 stacks_avail = 0; + int err = 0; lockdep_assert_held(&kbdev->hwaccess_lock); @@ -925,6 +920,15 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK); } + /* + * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores + * are vulnerable to corruption if gpu is lost + */ + if (kbase_is_gpu_lost(kbdev)) { + err = -EIO; + break; + } + /* mask off ready from trans in case transitions finished * between the register reads */ @@ -978,9 +982,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) case KBASE_SHADERS_PEND_ON_CORESTACK_ON: if (!shaders_trans && shaders_ready == backend->shaders_avail) { - KBASE_TRACE_ADD(kbdev, - PM_CORES_CHANGE_AVAILABLE, - NULL, NULL, 0u, (u32)shaders_ready); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready); backend->pm_shaders_core_mask = shaders_ready; backend->hwcnt_desired = true; if (backend->hwcnt_disabled) { @@ -1017,6 +1019,10 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) ; } else if (!backend->shaders_desired) { if (kbdev->pm.backend.protected_transition_override || +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev) || +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ !stt->configured_ticks || WARN_ON(stt->cancel_queued)) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; @@ -1076,6 +1082,11 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; } else if (stt->remaining_ticks == 0) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + } else if (kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev)) { + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } break; @@ -1129,17 +1140,12 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, shaders_ready & ~backend->shaders_avail, ACTION_PWROFF); backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - KBASE_TRACE_ADD(kbdev, - PM_CORES_CHANGE_AVAILABLE, - NULL, NULL, 0u, - (u32)(shaders_ready & ~backend->shaders_avail)); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail)); } else { kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, shaders_ready, ACTION_PWROFF); - KBASE_TRACE_ADD(kbdev, - PM_CORES_CHANGE_AVAILABLE, - NULL, NULL, 0u, 0u); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; } @@ -1188,6 +1194,8 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) backend->shaders_state)); } while (backend->shaders_state != prev_state); + + return err; } static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) @@ -1280,24 +1288,29 @@ void kbase_pm_update_state(struct kbase_device *kbdev) if (!kbdev->pm.backend.gpu_powered) return; /* Do nothing if the GPU is off */ - kbase_pm_l2_update_state(kbdev); - kbase_pm_shaders_update_state(kbdev); + if (kbase_pm_l2_update_state(kbdev)) + return; + + if (kbase_pm_shaders_update_state(kbdev)) + return; /* If the shaders just turned off, re-invoke the L2 state machine, in * case it was waiting for the shaders to turn off before powering down * the L2. */ if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF && - kbdev->pm.backend.shaders_state == KBASE_SHADERS_OFF_CORESTACK_OFF) - kbase_pm_l2_update_state(kbdev); + kbdev->pm.backend.shaders_state == + KBASE_SHADERS_OFF_CORESTACK_OFF) { + if (kbase_pm_l2_update_state(kbdev)) + return; + } if (kbase_pm_is_in_desired_state_nolock(kbdev)) { - KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, - true, kbdev->pm.backend.shaders_avail); + KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail); kbase_pm_trace_power_state(kbdev); - KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); } } @@ -1576,7 +1589,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) kbdev->poweroff_pending = false; - KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); + KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u); if (is_resume && kbdev->pm.backend.callback_power_resume) { kbdev->pm.backend.callback_power_resume(kbdev); @@ -1639,7 +1652,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) return true; } - KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); + KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u); /* Disable interrupts. This also clears any outstanding interrupts */ kbase_pm_disable_interrupts(kbdev); @@ -1660,6 +1673,9 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) /* The GPU power may be turned off from this point */ kbdev->pm.backend.gpu_powered = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbdev->pm.backend.callback_power_off) kbdev->pm.backend.callback_power_off(kbdev); @@ -1713,9 +1729,9 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } -static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) +static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) { - kbdev->hw_quirks_jm = kbase_reg_read(kbdev, + u32 hw_quirks_jm = kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG)); if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) { @@ -1730,11 +1746,17 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) */ if (coherency_features == COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { - kbdev->hw_quirks_jm |= (COHERENCY_ACE_LITE | + hw_quirks_jm |= (COHERENCY_ACE_LITE | COHERENCY_ACE) << JM_FORCE_COHERENCY_FEATURES_SHIFT; } } + + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + + kbdev->hw_quirks_jm = hw_quirks_jm; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { int default_idvs_group_size = 0xF; u32 tmp; @@ -1756,40 +1778,58 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) #define MANUAL_POWER_CONTROL ((u32)(1 << 8)) if (corestack_driver_control) kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; + + return 0; } -static void kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) +static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) { - kbdev->hw_quirks_sc = kbase_reg_read(kbdev, + u32 hw_quirks_sc = kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)); + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ - kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; + hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ - kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; + hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) - kbdev->hw_quirks_sc |= SC_VAR_ALGORITHM; + hw_quirks_sc |= SC_VAR_ALGORITHM; if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING)) - kbdev->hw_quirks_sc |= SC_TLS_HASH_ENABLE; + hw_quirks_sc |= SC_TLS_HASH_ENABLE; + + kbdev->hw_quirks_sc = hw_quirks_sc; + + return 0; } -static void kbase_set_tiler_quirks(struct kbase_device *kbdev) +static int kbase_set_tiler_quirks(struct kbase_device *kbdev) { - kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, + u32 hw_quirks_tiler = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG)); + + if (kbase_is_gpu_lost(kbdev)) + return -EIO; + /* Set tiler clock gate override if required */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) - kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; + hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; + + kbdev->hw_quirks_tiler = hw_quirks_tiler; + + return 0; } -static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) +static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) { struct device_node *np = kbdev->dev->of_node; const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + int error = 0; kbdev->hw_quirks_jm = 0; kbdev->hw_quirks_sc = 0; @@ -1802,7 +1842,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) "Found quirks_jm = [0x%x] in Devicetree\n", kbdev->hw_quirks_jm); } else { - kbase_set_jm_quirks(kbdev, prod_id); + error = kbase_set_jm_quirks(kbdev, prod_id); + if (error) + return error; } if (!of_property_read_u32(np, "quirks_sc", @@ -1811,7 +1853,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) "Found quirks_sc = [0x%x] in Devicetree\n", kbdev->hw_quirks_sc); } else { - kbase_set_sc_quirks(kbdev, prod_id); + error = kbase_set_sc_quirks(kbdev, prod_id); + if (error) + return error; } if (!of_property_read_u32(np, "quirks_tiler", @@ -1820,7 +1864,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) "Found quirks_tiler = [0x%x] in Devicetree\n", kbdev->hw_quirks_tiler); } else { - kbase_set_tiler_quirks(kbdev); + error = kbase_set_tiler_quirks(kbdev); + if (error) + return error; } if (!of_property_read_u32(np, "quirks_mmu", @@ -1829,8 +1875,10 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) "Found quirks_mmu = [0x%x] in Devicetree\n", kbdev->hw_quirks_mmu); } else { - kbase_set_mmu_quirks(kbdev); + error = kbase_set_mmu_quirks(kbdev); } + + return error; } static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) @@ -1892,7 +1940,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) struct kbasep_reset_timeout_data rtdata; int ret; - KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0); KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); @@ -1933,8 +1981,9 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) /* No interrupt has been received - check if the RAWSTAT register says * the reset has completed */ - if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & - RESET_COMPLETED) { + if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & + RESET_COMPLETED) + || kbase_is_gpu_lost(kbdev)) { /* The interrupt is set in the RAWSTAT; this suggests that the * interrupts are not getting to the CPU */ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); @@ -1947,7 +1996,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) * reset */ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT); - KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); @@ -2016,8 +2065,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) /* The cores should be made unavailable due to the reset */ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, - NULL, 0u, (u32)0u); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); /* Soft reset the GPU */ @@ -2031,8 +2079,11 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) if (err) goto exit; - if (flags & PM_HW_ISSUES_DETECT) - kbase_pm_hw_issues_detect(kbdev); + if (flags & PM_HW_ISSUES_DETECT) { + err = kbase_pm_hw_issues_detect(kbdev); + if (err) + goto exit; + } kbase_pm_hw_issues_apply(kbdev); kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h index 8d19318..95f10e0 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h +++ b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -682,4 +682,29 @@ extern bool corestack_driver_control; * Return: true if l2 need to power on */ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); + +/** + * kbase_pm_lock - Lock all necessary mutexes to perform PM actions + * + * @kbdev: Device pointer + * + * This function locks correct mutexes independent of GPU architecture. + */ +static inline void kbase_pm_lock(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->js_data.runpool_mutex); + mutex_lock(&kbdev->pm.lock); +} + +/** + * kbase_pm_unlock - Unlock mutexes locked by kbase_pm_lock + * + * @kbdev: Device pointer + */ +static inline void kbase_pm_unlock(struct kbase_device *kbdev) +{ + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&kbdev->js_data.runpool_mutex); +} + #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c index ae494b0..de3babe 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -31,6 +31,7 @@ #include <backend/gpu/mali_kbase_pm_internal.h> #include <backend/gpu/mali_kbase_jm_rb.h> #include <backend/gpu/mali_kbase_pm_defs.h> +#include <mali_linux_trace.h> /* When VSync is being hit aim for utilisation between 70-90% */ #define KBASE_PM_VSYNC_MIN_UTILISATION 70 @@ -284,8 +285,11 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) active_cl_ctx[device_nr] = 1; } else { kbdev->pm.backend.metrics.active_gl_ctx[js] = 1; + trace_sysgraph(SGR_ACTIVE, 0, js); } kbdev->pm.backend.metrics.gpu_active = true; + } else { + trace_sysgraph(SGR_INACTIVE, 0, js); } } } diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c index 7d9bb03..17ed21e 100644 --- a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c +++ b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -102,6 +102,8 @@ void kbase_pm_update_active(struct kbase_device *kbdev) * when there are contexts active */ KBASE_DEBUG_ASSERT(pm->active_count == 0); + pm->backend.poweron_required = false; + /* Request power off */ if (pm->backend.gpu_powered) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -157,8 +159,7 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); if (kbdev->pm.backend.shaders_desired != shaders_desired) { - KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, - (u32)kbdev->pm.backend.shaders_desired); + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); kbdev->pm.backend.shaders_desired = shaders_desired; kbase_pm_update_state(kbdev); @@ -199,22 +200,20 @@ KBASE_EXPORT_TEST_API(kbase_pm_get_policy); void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *new_policy) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; const struct kbase_pm_policy *old_policy; unsigned long flags; KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(new_policy != NULL); - KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); + KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); /* During a policy change we pretend the GPU is active */ /* A suspend won't happen here, because we're in a syscall from a * userspace thread */ kbase_pm_context_active(kbdev); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); /* Remove the policy to prevent IRQ handlers from working on it */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -222,13 +221,11 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, kbdev->pm.backend.pm_current_policy = NULL; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, - old_policy->id); + KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, old_policy->id); if (old_policy->term) old_policy->term(kbdev); - KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, - new_policy->id); + KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id); if (new_policy->init) new_policy->init(kbdev); @@ -242,8 +239,7 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, kbase_pm_update_active(kbdev); kbase_pm_update_cores_state(kbdev); - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); /* Now the policy change is finished, we release our fake context active * reference */ diff --git a/mali_kbase/backend/gpu/mali_kbase_time.c b/mali_kbase/backend/gpu/mali_kbase_time.c index 057bf10..cb10518 100644 --- a/mali_kbase/backend/gpu/mali_kbase_time.c +++ b/mali_kbase/backend/gpu/mali_kbase_time.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016,2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -26,7 +26,7 @@ #include <backend/gpu/mali_kbase_pm_internal.h> void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, - u64 *system_time, struct timespec *ts) + u64 *system_time, struct timespec64 *ts) { u32 hi1, hi2; @@ -60,7 +60,11 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, /* Record the CPU's idea of current time */ if (ts != NULL) - getrawmonotonic(ts); +#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE) + *ts = ktime_to_timespec64(ktime_get_raw()); +#else + ktime_get_raw_ts64(ts); +#endif kbase_pm_release_gpu_cycle_counter(kbdev); } diff --git a/mali_kbase/build.bp b/mali_kbase/build.bp index 94189b1..51aeecd 100644 --- a/mali_kbase/build.bp +++ b/mali_kbase/build.bp @@ -37,9 +37,6 @@ bob_defaults { buslog: { kbuild_options: ["CONFIG_MALI_BUSLOG=y"], }, - cinstr_job_dump: { - kbuild_options: ["CONFIG_MALI_JOB_DUMP=y"], - }, cinstr_vector_dump: { kbuild_options: ["CONFIG_MALI_VECTOR_DUMP=y"], }, @@ -49,6 +46,9 @@ bob_defaults { mali_gator_support: { kbuild_options: ["CONFIG_MALI_GATOR_SUPPORT=y"], }, + mali_midgard_enable_trace: { + kbuild_options: ["CONFIG_MALI_MIDGARD_ENABLE_TRACE=y"], + }, mali_system_trace: { kbuild_options: ["CONFIG_MALI_SYSTEM_TRACE=y"], }, @@ -64,6 +64,12 @@ bob_defaults { mali_dma_buf_legacy_compat: { kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"], }, + mali_arbiter_support: { + kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"], + }, + mali_gem5_build: { + kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"], + }, kbuild_options: [ "MALI_UNIT_TEST={{.unit_test_code}}", "MALI_CUSTOMER_RELEASE={{.release}}", @@ -92,6 +98,8 @@ bob_kernel_module { "platform/*/*.h", "platform/*/Kbuild", "thirdparty/*.c", + "debug/*.c", + "debug/*.h", "device/*.c", "device/*.h", "gpu/*.c", @@ -139,6 +147,8 @@ bob_kernel_module { gpu_has_job_manager: { srcs: [ "context/backend/*_jm.c", + "debug/backend/*_jm.c", + "debug/backend/*_jm.h", "device/backend/*_jm.c", "gpu/backend/*_jm.c", "gpu/backend/*_jm.h", @@ -153,6 +163,8 @@ bob_kernel_module { "csf/*.c", "csf/*.h", "csf/Kbuild", + "debug/backend/*_csf.c", + "debug/backend/*_csf.h", "device/backend/*_csf.c", "gpu/backend/*_csf.c", "gpu/backend/*_csf.h", @@ -160,5 +172,12 @@ bob_kernel_module { "mmu/backend/*_csf.c", ], }, + mali_arbiter_support: { + srcs: [ + "arbiter/*.c", + "arbiter/*.h", + "arbiter/Kbuild", + ], + }, defaults: ["mali_kbase_shared_config_defaults"], } diff --git a/mali_kbase/context/mali_kbase_context.c b/mali_kbase/context/mali_kbase_context.c index a539edb..93fe431 100644 --- a/mali_kbase/context/mali_kbase_context.c +++ b/mali_kbase/context/mali_kbase_context.c @@ -80,7 +80,9 @@ int kbase_context_common_init(struct kbase_context *kctx) mutex_lock(&kctx->kbdev->kctx_list_lock); list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); - /* Trace with the AOM tracepoint even in CSF for dumping */ + + KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id, + kctx->kbdev->gpu_props.props.raw_props.gpu_id); KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id, (u32)(kctx->tgid)); mutex_unlock(&kctx->kbdev->kctx_list_lock); @@ -107,19 +109,21 @@ void kbase_context_common_term(struct kbase_context *kctx) WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); mutex_lock(&kctx->kbdev->kctx_list_lock); - /* Trace with the AOM tracepoint even in CSF for dumping */ + + KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id); + KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx); list_del(&kctx->kctx_list_link); mutex_unlock(&kctx->kbdev->kctx_list_lock); - KBASE_TRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); + KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); /* Flush the timeline stream, so the user can see the termination * tracepoints being fired. * The "if" statement below is for optimization. It is safe to call * kbase_timeline_streams_flush when timeline is disabled. */ - if (atomic_read(&kctx->kbdev->timeline_is_enabled) != 0) + if (atomic_read(&kctx->kbdev->timeline_flags) != 0) kbase_timeline_streams_flush(kctx->kbdev->timeline); vfree(kctx); diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h new file mode 100644 index 0000000..d534f30 --- /dev/null +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h @@ -0,0 +1,170 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** + */ + +/* + * The purpose of this header file is just to contain a list of trace code + * identifiers + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT + * DESCRIBED IN mali_kbase_debug_ktrace_codes.h + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + + /* + * Job Slot management events + */ + /* info_val==irq rawstat at start */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ), + /* info_val==jobs processed */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END), + /* In the following: + * + * - ctx is set if a corresponding job found (NULL otherwise, e.g. some + * soft-stop cases) + * - uatom==kernel-side mapped uatom address (for correlation with + * user-side) + */ + /* info_val==exit code; gpu_addr==chain gpuaddr */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE), + /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of + * affinity + */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT), + /* gpu_addr is as follows: + * - If JS_STATUS active after soft-stop, val==gpu addr written to + * JS_HEAD on submit + * - otherwise gpu_addr==0 + */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), + /* gpu_addr==JS_TAIL read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), + /* gpu_addr is as follows: + * - If JS_STATUS active before soft-stop, val==JS_HEAD + * - otherwise gpu_addr==0 + */ + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), + KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), + KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), + /* info_val == is_scheduled */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), + /* info_val == is_scheduled */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE), + /* info_val == nr jobs submitted */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), + /* gpu_addr==JS_HEAD_NEXT last written */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), + KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), + /* + * Job dispatch events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==0, info_val==0, uatom==0 */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), + /* + * Scheduler Core events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB), + /* gpu_addr==last value written/would be written to JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), + /* info_val == the ctx attribute now on ctx */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), + /* info_val == the ctx attribute now on runpool */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), + /* info_val == the ctx attribute now off ctx */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), + /* info_val == the ctx attribute now off runpool */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), + /* + * Scheduler Policy events + */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), + /* info_val == whether it was evicted */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), + /* gpu_addr==JS_HEAD to write if the job were run */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h new file mode 100644 index 0000000..55b66ad --- /dev/null +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h @@ -0,0 +1,75 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ +#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ + +/** + * DOC: KTrace version history, JM variant + * 1.0: + * - Original version (implicit, header did not carry version information) + * 2.0: + * - Introduced version information into the header + * - some changes of parameter names in header + * - trace now uses all 64-bits of info_val + * - Non-JM specific parts moved to using info_val instead of refcount/gpu_addr + */ +#define KBASE_KTRACE_VERSION_MAJOR 2 +#define KBASE_KTRACE_VERSION_MINOR 0 + +/* indicates if the trace message has a valid refcount member */ +#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) +/* indicates if the trace message has a valid jobslot member */ +#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1) +/* indicates if the trace message has valid atom related info. */ +#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) + + +/** + * struct kbase_ktrace_backend - backend specific part of a trace message + * + * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. + * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags + * @gpu_addr: GPU address, usually of the job-chain represented by an atom. + * @atom_number: id of the atom for which trace message was added. Only valid + * if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags + * @code: Identifies the event, refer to enum kbase_ktrace_code. + * @flags: indicates information about the trace message itself. Used + * during dumping of the message. + * @jobslot: job-slot for which trace message was added, valid only for + * job-slot management events. + * @refcount: reference count for the context, valid for certain events + * related to scheduler core and policy. + */ +struct kbase_ktrace_backend { + /* Place 64 and 32-bit members together */ + u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + u64 gpu_addr; + int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + /* Pack smaller members together */ + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 jobslot; + u8 refcount; +}; + +#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c new file mode 100644 index 0000000..e651a09 --- /dev/null +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c @@ -0,0 +1,113 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include <mali_kbase.h> +#include "debug/mali_kbase_debug_ktrace_internal.h" +#include "debug/backend/mali_kbase_debug_ktrace_jm.h" + +#if KBASE_KTRACE_TARGET_RBUF + +void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) +{ + *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + "katom,gpu_addr,jobslot,refcount"), 0); +} + +void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz, s32 *written) +{ + /* katom */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "atom %d (ud: 0x%llx 0x%llx)", + trace_msg->backend.atom_number, + trace_msg->backend.atom_udata[0], + trace_msg->backend.atom_udata[1]), 0); + + /* gpu_addr */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ",%.8llx,", trace_msg->backend.gpu_addr), 0); + else + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ",,"), 0); + + /* jobslot */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", trace_msg->backend.jobslot), 0); + + *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + ","), 0); + + /* refcount */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", trace_msg->backend.refcount), 0); +} + +void kbasep_ktrace_add_jm(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_context *kctx, + struct kbase_jd_atom *katom, u64 gpu_addr, + kbase_ktrace_flag_t flags, int refcount, int jobslot, + u64 info_val) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Fill the common part of the message (including backend.flags) */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, + info_val); + + /* Indicate to the common code that backend-specific parts will be + * valid + */ + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; + + /* Fill the JM-specific parts of the message */ + if (katom) { + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM; + + trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom); + trace_msg->backend.atom_udata[0] = katom->udata.blob[0]; + trace_msg->backend.atom_udata[1] = katom->udata.blob[1]; + } + + trace_msg->backend.gpu_addr = gpu_addr; + trace_msg->backend.jobslot = jobslot; + /* Clamp refcount */ + trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h new file mode 100644 index 0000000..c1bacf9 --- /dev/null +++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h @@ -0,0 +1,362 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_JM_H_ +#define _KBASE_DEBUG_KTRACE_JM_H_ + +/* + * KTrace target for internal ringbuffer + */ +#if KBASE_KTRACE_TARGET_RBUF +/** + * kbasep_ktrace_add_jm - internal function to add trace about Job Management + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @flags: flags about the message + * @refcount: reference count information to add to the trace + * @jobslot: jobslot information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead. + */ +void kbasep_ktrace_add_jm(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_context *kctx, + struct kbase_jd_atom *katom, u64 gpu_addr, + kbase_ktrace_flag_t flags, int refcount, int jobslot, + u64 info_val); + +#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, 0) + +#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ + jobslot, info_val) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, \ + info_val) + +#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ + refcount) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, 0) +#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, \ + info_val) + +#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, 0, 0, 0, info_val) + +#else /* KBASE_KTRACE_TARGET_RBUF */ +#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ + jobslot, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ + refcount) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(refcount);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ + info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * KTrace target for Linux's ftrace + */ +#if KBASE_KTRACE_TARGET_FTRACE +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + trace_mali_##code(jobslot, 0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, jobslot, info_val) \ + trace_mali_##code(jobslot, info_val) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ + gpu_addr, refcount) \ + trace_mali_##code(refcount, 0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + trace_mali_##code(refcount, info_val) + +#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ + info_val) \ + trace_mali_##code(gpu_addr, info_val) +#else /* KBASE_KTRACE_TARGET_FTRACE */ +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, jobslot, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ + gpu_addr, refcount) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(refcount);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ + info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +/* + * Master set of macros to route KTrace to any of the targets + */ + +/** + * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @jobslot: jobslot information to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __jobslot = jobslot; \ + KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ + KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @jobslot: jobslot information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ + jobslot, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __jobslot = jobslot; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @refcount: reference count information to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ + refcount) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __refcount = refcount; \ + KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ + KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount, + * and info + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @refcount: reference count information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __refcount = refcount; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount) + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ + } while (0) + +#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */ diff --git a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h new file mode 100644 index 0000000..d964e5a --- /dev/null +++ b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h @@ -0,0 +1,150 @@ +/* + * + * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * NOTE: This must **only** be included through mali_linux_trace.h, + * otherwise it will fail to setup tracepoints correctly + */ + +#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ + +DECLARE_EVENT_CLASS(mali_jm_slot_template, + TP_PROTO(int jobslot, u64 info_val), + TP_ARGS(jobslot, info_val), + TP_STRUCT__entry( + __field(unsigned int, jobslot) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->jobslot = jobslot; + __entry->info_val = info_val; + ), + TP_printk("jobslot=%u info=0x%llx", __entry->jobslot, __entry->info_val) +); + +#define DEFINE_MALI_JM_SLOT_EVENT(name) \ +DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ + TP_PROTO(int jobslot, u64 info_val), \ + TP_ARGS(jobslot, info_val)) +DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); +DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); +DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); +DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1); +DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT); +DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER); +DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); +DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT); +DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); +DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); +DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); +#undef DEFINE_MALI_JM_SLOT_EVENT + +DECLARE_EVENT_CLASS(mali_jm_refcount_template, + TP_PROTO(int refcount, u64 info_val), + TP_ARGS(refcount, info_val), + TP_STRUCT__entry( + __field(unsigned int, refcount) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->info_val = info_val; + ), + TP_printk("refcount=%u info=0x%llx", __entry->refcount, __entry->info_val) +); + +#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ +DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ + TP_PROTO(int refcount, u64 info_val), \ + TP_ARGS(refcount, info_val)) +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); +#undef DEFINE_MALI_JM_REFCOUNT_EVENT + +DECLARE_EVENT_CLASS(mali_jm_add_template, + TP_PROTO(u64 gpu_addr, u64 info_val), + TP_ARGS(gpu_addr, info_val), + TP_STRUCT__entry( + __field(u64, gpu_addr) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->gpu_addr = gpu_addr; + __entry->info_val = info_val; + ), + TP_printk("gpu_addr=0x%llx info=0x%llx", __entry->gpu_addr, __entry->info_val) +); + +#define DEFINE_MALI_JM_ADD_EVENT(name) \ +DEFINE_EVENT(mali_jm_add_template, mali_##name, \ + TP_PROTO(u64 gpu_addr, u64 info_val), \ + TP_ARGS(gpu_addr, info_val)) +DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); +DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); +DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); +DEFINE_MALI_JM_ADD_EVENT(JD_DONE); +DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL); +DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT); +DEFINE_MALI_JM_ADD_EVENT(JM_IRQ); +DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END); +DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS); +DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE); +DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET); +DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); +#undef DEFINE_MALI_JM_ADD_EVENT + +#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/ diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.c b/mali_kbase/debug/mali_kbase_debug_ktrace.c new file mode 100644 index 0000000..6322abb --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_ktrace.c @@ -0,0 +1,342 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include <mali_kbase.h> +#include "debug/mali_kbase_debug_ktrace_internal.h" + +int kbase_ktrace_init(struct kbase_device *kbdev) +{ +#if KBASE_KTRACE_TARGET_RBUF + struct kbase_ktrace_msg *rbuf; + + /* See also documentation of enum kbase_ktrace_code */ + compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) || + KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)), + "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT"); + + rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); + + if (!rbuf) + return -EINVAL; + + kbdev->ktrace.rbuf = rbuf; + spin_lock_init(&kbdev->ktrace.lock); +#endif /* KBASE_KTRACE_TARGET_RBUF */ + return 0; +} + +void kbase_ktrace_term(struct kbase_device *kbdev) +{ +#if KBASE_KTRACE_TARGET_RBUF + kfree(kbdev->ktrace.rbuf); +#endif /* KBASE_KTRACE_TARGET_RBUF */ +} + +void kbase_ktrace_hook_wrapper(void *param) +{ + struct kbase_device *kbdev = (struct kbase_device *)param; + + KBASE_KTRACE_DUMP(kbdev); +} + +#if KBASE_KTRACE_TARGET_RBUF + +static const char * const kbasep_ktrace_code_string[] = { + /* + * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ARRAY + */ +#define KBASE_KTRACE_CODE_MAKE_CODE(X) # X +#include "debug/mali_kbase_debug_ktrace_codes.h" +#undef KBASE_KTRACE_CODE_MAKE_CODE +}; + +static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written) +{ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + "secs,thread_id,cpu,code,kctx,"), 0); + + kbasep_ktrace_backend_format_header(buffer, sz, &written); + + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + ",info_val,ktrace_version=%u.%u", + KBASE_KTRACE_VERSION_MAJOR, + KBASE_KTRACE_VERSION_MINOR), 0); + + buffer[sz - 1] = 0; +} + +static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz) +{ + s32 written = 0; + + /* Initial part of message: + * + * secs,thread_id,cpu,code,kctx, + */ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + "%d.%.6d,%d,%d,%s,%p,", + (int)trace_msg->timestamp.tv_sec, + (int)(trace_msg->timestamp.tv_nsec / 1000), + trace_msg->thread_id, trace_msg->cpu, + kbasep_ktrace_code_string[trace_msg->backend.code], + trace_msg->kctx), 0); + + /* Backend parts */ + kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, + &written); + + /* Rest of message: + * + * ,info_val + * + * Note that the last column is empty, it's simply to hold the ktrace + * version in the header + */ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + ",0x%.16llx", + (unsigned long long)trace_msg->info_val), 0); + buffer[sz - 1] = 0; +} + +static void kbasep_ktrace_dump_msg(struct kbase_device *kbdev, + struct kbase_ktrace_msg *trace_msg) +{ + char buffer[KTRACE_DUMP_MESSAGE_SIZE]; + + lockdep_assert_held(&kbdev->ktrace.lock); + + kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); + dev_dbg(kbdev->dev, "%s", buffer); +} + +struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace) +{ + struct kbase_ktrace_msg *trace_msg; + + lockdep_assert_held(&ktrace->lock); + + trace_msg = &ktrace->rbuf[ktrace->next_in]; + + /* Update the ringbuffer indices */ + ktrace->next_in = (ktrace->next_in + 1) & KBASE_KTRACE_MASK; + if (ktrace->next_in == ktrace->first_out) + ktrace->first_out = (ktrace->first_out + 1) & KBASE_KTRACE_MASK; + + return trace_msg; +} +void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, + struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val) +{ + lockdep_assert_held(&ktrace->lock); + + trace_msg->thread_id = task_pid_nr(current); + trace_msg->cpu = task_cpu(current); + + ktime_get_real_ts64(&trace_msg->timestamp); + + trace_msg->kctx = kctx; + + trace_msg->info_val = info_val; + trace_msg->backend.code = code; + trace_msg->backend.flags = flags; +} + +void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Fill the common part of the message (including backend.flags) */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, + info_val); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +static void kbasep_ktrace_clear_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ktrace.lock); + kbdev->ktrace.first_out = kbdev->ktrace.next_in; +} +void kbasep_ktrace_clear(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + kbasep_ktrace_clear_locked(kbdev); + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); +} + +void kbasep_ktrace_dump(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 start; + u32 end; + char buffer[KTRACE_DUMP_MESSAGE_SIZE] = "Dumping trace:\n"; + + kbasep_ktrace_format_header(buffer, sizeof(buffer), strlen(buffer)); + dev_dbg(kbdev->dev, "%s", buffer); + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + start = kbdev->ktrace.first_out; + end = kbdev->ktrace.next_in; + + while (start != end) { + struct kbase_ktrace_msg *trace_msg = &kbdev->ktrace.rbuf[start]; + + kbasep_ktrace_dump_msg(kbdev, trace_msg); + + start = (start + 1) & KBASE_KTRACE_MASK; + } + dev_dbg(kbdev->dev, "TRACE_END"); + + kbasep_ktrace_clear_locked(kbdev); + + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); +} + +#ifdef CONFIG_DEBUG_FS +struct trace_seq_state { + struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE]; + u32 start; + u32 end; +}; + +static void *kbasep_ktrace_seq_start(struct seq_file *s, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (*pos == 0) + /* See Documentation/filesystems/seq_file.txt */ + return SEQ_START_TOKEN; + + if (*pos > KBASE_KTRACE_SIZE) + return NULL; + i = state->start + *pos; + if ((state->end >= state->start && i >= state->end) || + i >= state->end + KBASE_KTRACE_SIZE) + return NULL; + + i &= KBASE_KTRACE_MASK; + + return &state->trace_buf[i]; +} + +static void kbasep_ktrace_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_ktrace_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (data != SEQ_START_TOKEN) + (*pos)++; + + i = (state->start + *pos) & KBASE_KTRACE_MASK; + if (i == state->end) + return NULL; + + return &state->trace_buf[i]; +} + +static int kbasep_ktrace_seq_show(struct seq_file *s, void *data) +{ + struct kbase_ktrace_msg *trace_msg = data; + char buffer[KTRACE_DUMP_MESSAGE_SIZE]; + + /* If this is the start, print a header */ + if (data == SEQ_START_TOKEN) + kbasep_ktrace_format_header(buffer, sizeof(buffer), 0); + else + kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); + + seq_printf(s, "%s\n", buffer); + return 0; +} + +static const struct seq_operations kbasep_ktrace_seq_ops = { + .start = kbasep_ktrace_seq_start, + .next = kbasep_ktrace_seq_next, + .stop = kbasep_ktrace_seq_stop, + .show = kbasep_ktrace_seq_show, +}; + +static int kbasep_ktrace_debugfs_open(struct inode *inode, struct file *file) +{ + struct kbase_device *kbdev = inode->i_private; + unsigned long flags; + + struct trace_seq_state *state; + + state = __seq_open_private(file, &kbasep_ktrace_seq_ops, + sizeof(*state)); + if (!state) + return -ENOMEM; + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + state->start = kbdev->ktrace.first_out; + state->end = kbdev->ktrace.next_in; + memcpy(state->trace_buf, kbdev->ktrace.rbuf, sizeof(state->trace_buf)); + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); + + return 0; +} + +static const struct file_operations kbasep_ktrace_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_ktrace_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_trace", 0444, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_ktrace_debugfs_fops); +} +#endif /* CONFIG_DEBUG_FS */ + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#ifdef CONFIG_DEBUG_FS +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.h b/mali_kbase/debug/mali_kbase_debug_ktrace.h new file mode 100644 index 0000000..0dd8b7a --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_ktrace.h @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * DOC: Kbase's own trace, 'KTrace' + * + * Low overhead trace specific to kbase, aimed at: + * - common use-cases for tracing kbase specific functionality to do with + * running work on the GPU + * - easy 1-line addition of new types of trace + * + * KTrace can be recorded in one or more of the following targets: + * - KBASE_KTRACE_TARGET_RBUF: low overhead ringbuffer protected by an + * irq-spinlock, output available via dev_dbg() and debugfs file + * - KBASE_KTRACE_TARGET_FTRACE: ftrace based tracepoints under 'mali' events + */ + +#ifndef _KBASE_DEBUG_KTRACE_H_ +#define _KBASE_DEBUG_KTRACE_H_ + +#include "debug/backend/mali_kbase_debug_ktrace_jm.h" + +/** + * kbase_ktrace_init - initialize kbase ktrace. + * @kbdev: kbase device + */ +int kbase_ktrace_init(struct kbase_device *kbdev); + +/** + * kbase_ktrace_term - terminate kbase ktrace. + * @kbdev: kbase device + */ +void kbase_ktrace_term(struct kbase_device *kbdev); + +/** + * kbase_ktrace_hook_wrapper - wrapper so that dumping ktrace can be done via a + * callback. + * @param: kbase device, cast to void pointer + */ +void kbase_ktrace_hook_wrapper(void *param); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if + * the selected targets support it. + * @kbdev: kbase device + * + * There is no matching 'term' call, debugfs_remove_recursive() is sufficient. + */ +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev); +#endif /* CONFIG_DEBUG_FS */ + +/* + * KTrace target for internal ringbuffer + */ +#if KBASE_KTRACE_TARGET_RBUF +/** + * kbasep_ktrace_add - internal function to add trace to the ringbuffer. + * @kbdev: kbase device + * @code: ktrace code + * @kctx: kbase context, or NULL if no context + * @flags: flags about the message + * @info_val: generic information about @code to add to the trace + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD() instead. + */ +void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val); + +/** + * kbasep_ktrace_clear - clear the trace ringbuffer + * @kbdev: kbase device + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_CLEAR() instead. + */ +void kbasep_ktrace_clear(struct kbase_device *kbdev); + +/** + * kbasep_ktrace_dump - dump ktrace ringbuffer to dev_dbg(), then clear it + * @kbdev: kbase device + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_DUMP() instead. + */ +void kbasep_ktrace_dump(struct kbase_device *kbdev); + +#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ + kbasep_ktrace_add(kbdev, KBASE_KTRACE_CODE(code), kctx, 0, \ + info_val) \ + +#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ + kbasep_ktrace_clear(kbdev) + +#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ + kbasep_ktrace_dump(kbdev) + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(code); \ + CSTD_UNUSED(kctx); \ + CSTD_UNUSED(info_val); \ + CSTD_NOP(0); \ + } while (0) + +#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) +#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * KTrace target for Linux's ftrace + */ +#if KBASE_KTRACE_TARGET_FTRACE +#include "mali_linux_trace.h" + +#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ + trace_mali_##code(info_val) + +#else /* KBASE_KTRACE_TARGET_FTRACE */ +#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(code); \ + CSTD_UNUSED(kctx); \ + CSTD_UNUSED(info_val); \ + CSTD_NOP(0); \ + } while (0) +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +/* No 'clear' implementation for ftrace yet */ +#define KBASE_KTRACE_FTRACE_CLEAR(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) + +/* No 'dump' implementation for ftrace yet */ +#define KBASE_KTRACE_FTRACE_DUMP(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) + +/* + * Master set of macros to route KTrace to any of the targets + */ + +/** + * KBASE_KTRACE_ADD - Add trace values + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD(kbdev, code, kctx, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, __info_val); \ + KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_CLEAR - Clear the trace, if applicable to the target(s) + * @kbdev: kbase device + */ +#define KBASE_KTRACE_CLEAR(kbdev) \ + do { \ + KBASE_KTRACE_RBUF_CLEAR(kbdev); \ + KBASE_KTRACE_FTRACE_CLEAR(kbdev); \ + } while (0) + +/** + * KBASE_KTRACE_DUMP - Dump the trace, if applicable to the target(s) + * @kbdev: kbase device + */ +#define KBASE_KTRACE_DUMP(kbdev) \ + do { \ + KBASE_KTRACE_RBUF_DUMP(kbdev); \ + KBASE_KTRACE_FTRACE_DUMP(kbdev); \ + } while (0) + +#endif /* _KBASE_DEBUG_KTRACE_H_ */ diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h new file mode 100644 index 0000000..364ed60 --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h @@ -0,0 +1,158 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** + */ + +/* + * The purpose of this header file is just to contain a list of trace code + * identifiers + * + * Each identifier is wrapped in a macro, so that its string form and enum form + * can be created + * + * Each macro is separated with a comma, to allow insertion into an array + * initializer or enum definition block. + * + * This allows automatic creation of an enum and a corresponding array of + * strings + * + * Before #including, the includer MUST #define KBASE_KTRACE_CODE_MAKE_CODE. + * After #including, the includer MUST #under KBASE_KTRACE_CODE_MAKE_CODE. + * + * e.g.: + * #define KBASE_KTRACE_CODE( X ) KBASE_KTRACE_CODE_ ## X + * typedef enum + * { + * #define KBASE_KTRACE_CODE_MAKE_CODE( X ) KBASE_KTRACE_CODE( X ) + * #include "mali_kbase_debug_ktrace_codes.h" + * #undef KBASE_KTRACE_CODE_MAKE_CODE + * } kbase_ktrace_code; + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE + * + * + * The use of the macro here is: + * - KBASE_KTRACE_CODE_MAKE_CODE( X ) + * + * Which produces: + * - For an enum, KBASE_KTRACE_CODE_X + * - For a string, "X" + * + * + * For example: + * - KBASE_KTRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: + * - KBASE_KTRACE_CODE_JM_JOB_COMPLETE for the enum + * - "JM_JOB_COMPLETE" for the string + * - To use it to trace an event, do: + * - KBASE_KTRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + + /* + * Core events + */ + /* no info_val */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), + /* no info_val */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), + /* info_val == bits cleared */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), + /* info_val == dump address */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), + + /* + * Power Management Events + */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), + KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), + /* info_val == kbdev->pm.active_count*/ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), + /* info_val == kbdev->pm.active_count*/ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_ON), + KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_OFF), + /* info_val == policy number, or -1 for "Already changing" */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_SET_POLICY), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), + /* info_val == policy number */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), + /* info_val == policy number */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), + + /* + * Context Scheduler events + */ + /* info_val == kctx->refcount */ + KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), + /* info_val == kctx->refcount */ + KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), + + +#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" + /* + * Unused code just to make it easier to not have a comma at the end. + * All other codes MUST come before this + */ + KBASE_KTRACE_CODE_MAKE_CODE(DUMMY) + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h new file mode 100644 index 0000000..d6baaf1 --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h @@ -0,0 +1,152 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_DEFS_H_ +#define _KBASE_DEBUG_KTRACE_DEFS_H_ + +/* Enable SW tracing when set */ +#if defined(CONFIG_MALI_MIDGARD_ENABLE_TRACE) || defined(CONFIG_MALI_SYSTEM_TRACE) +#define KBASE_KTRACE_ENABLE 1 +#endif + +#ifndef KBASE_KTRACE_ENABLE +#ifdef CONFIG_MALI_DEBUG +#define KBASE_KTRACE_ENABLE 1 +#else /* CONFIG_MALI_DEBUG */ +#define KBASE_KTRACE_ENABLE 0 +#endif /* CONFIG_MALI_DEBUG */ +#endif /* KBASE_KTRACE_ENABLE */ + +/* Select targets for recording of trace: + * + */ +#if KBASE_KTRACE_ENABLE + +#ifdef CONFIG_MALI_SYSTEM_TRACE +#define KBASE_KTRACE_TARGET_FTRACE 1 +#else /* CONFIG_MALI_SYSTEM_TRACE */ +#define KBASE_KTRACE_TARGET_FTRACE 0 +#endif /* CONFIG_MALI_SYSTEM_TRACE */ + +#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE +#define KBASE_KTRACE_TARGET_RBUF 1 +#else /* CONFIG_MALI_MIDGARD_ENABLE_TRACE*/ +#define KBASE_KTRACE_TARGET_RBUF 0 +#endif /* CONFIG_MALI_MIDGARD_ENABLE_TRACE */ + +#else /* KBASE_KTRACE_ENABLE */ +#define KBASE_KTRACE_TARGET_FTRACE 0 +#define KBASE_KTRACE_TARGET_RBUF 0 +#endif /* KBASE_KTRACE_ENABLE */ + +/* + * NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in + * the backend, since updates can be made to one backend in a way that doesn't + * affect the other. + * + * However, modifying the common part could require both backend versions to be + * updated. + */ + +#if KBASE_KTRACE_TARGET_RBUF +typedef u8 kbase_ktrace_flag_t; +typedef u8 kbase_ktrace_code_t; + +/* + * struct kbase_ktrace_backend - backend specific part of a trace message + * + * At the very least, this must contain a kbase_ktrace_code_t 'code' member and + * a kbase_ktrace_flag_t 'flags' member + */ +struct kbase_ktrace_backend; + +#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" + +/* Indicates if the trace message has backend related info. + * + * If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg + * as uninitialized, apart from the mandatory parts: + * - code + * - flags + */ +#define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7) + +#define KBASE_KTRACE_SHIFT 8 /* 256 entries */ +#define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT) +#define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1) + +#define KBASE_KTRACE_CODE(X) KBASE_KTRACE_CODE_ ## X + +/* Note: compiletime_assert() about this against kbase_ktrace_code_t is in + * kbase_ktrace_init() + */ +enum kbase_ktrace_code { + /* + * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ENUM + */ +#define KBASE_KTRACE_CODE_MAKE_CODE(X) KBASE_KTRACE_CODE(X) +#include <debug/mali_kbase_debug_ktrace_codes.h> +#undef KBASE_KTRACE_CODE_MAKE_CODE + /* Comma on its own, to extend the list */ + , + /* Must be the last in the enum */ + KBASE_KTRACE_CODE_COUNT +}; + +/** + * struct kbase_ktrace - object representing a trace message added to trace + * buffer trace_rbuf in &kbase_device + * @timestamp: CPU timestamp at which the trace message was added. + * @thread_id: id of the thread in the context of which trace message was + * added. + * @cpu: indicates which CPU the @thread_id was scheduled on when the + * trace message was added. + * @kctx: Pointer to the kbase context for which the trace message was + * added. Will be NULL for certain trace messages associated with + * the &kbase_device itself, such as power management events. + * Will point to the appropriate context corresponding to + * backend-specific events. + * @info_val: value specific to the type of event being traced. Refer to the + * specific code in enum kbase_ktrace_code + * @backend: backend-specific trace information. All backends must implement + * a minimum common set of members + */ +struct kbase_ktrace_msg { + struct timespec64 timestamp; + u32 thread_id; + u32 cpu; + void *kctx; + u64 info_val; + + struct kbase_ktrace_backend backend; +}; + +struct kbase_ktrace { + spinlock_t lock; + u16 first_out; + u16 next_in; + struct kbase_ktrace_msg *rbuf; +}; + +#endif /* KBASE_KTRACE_TARGET_RBUF */ +#endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */ diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h new file mode 100644 index 0000000..e450760 --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h @@ -0,0 +1,89 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_ +#define _KBASE_DEBUG_KTRACE_INTERNAL_H_ + +#if KBASE_KTRACE_TARGET_RBUF + +#define KTRACE_DUMP_MESSAGE_SIZE 256 + +/** + * kbasep_ktrace_backend_format_header - format the backend part of the header + * @buffer: buffer to write to + * @sz: size of @buffer in bytes + * @written: pointer to storage for updating bytes written so far to @buffer + * + * The backend must format only the non-common backend specific parts of the + * header. It must format them as though they were standalone. The caller will + * handle adding any delimiters around this. + */ +void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written); + +/** + * kbasep_ktrace_backend_format_msg - format the backend part of the message + * @trace_msg: ktrace message + * @buffer: buffer to write to + * @sz: size of @buffer in bytes + * @written: pointer to storage for updating bytes written so far to @buffer + * + * The backend must format only the non-common backend specific parts of the + * message. It must format them as though they were standalone. The caller will + * handle adding any delimiters around this. + * + * A caller may have the flags member of @trace_msg with + * %KBASE_KTRACE_FLAG_BACKEND clear. The backend must handle that setting + * appropriately. + */ +void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz, s32 *written); + + +/** + * kbasep_ktrace_reserve - internal function to reserve space for a ktrace + * message + * @ktrace: kbase device's ktrace + * + * This may also empty the oldest entry in the ringbuffer to make space. + */ +struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace); + +/** + * kbasep_ktrace_msg_init - internal function to initialize just the common + * part of a ktrace message + * @ktrace: kbase device's ktrace + * @trace_msg: ktrace message to initialize + * @code: ktrace code + * @kctx: kbase context, or NULL if no context + * @flags: flags about the message + * @info_val: generic information about @code to add to the trace + * + * The common part includes the mandatory parts of the backend part + */ +void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, + struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val); + +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +#endif /* _KBASE_DEBUG_KTRACE_INTERNAL_H_ */ diff --git a/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h new file mode 100644 index 0000000..18e4f7c --- /dev/null +++ b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h @@ -0,0 +1,99 @@ +/* + * + * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * NOTE: This must **only** be included through mali_linux_trace.h, + * otherwise it will fail to setup tracepoints correctly + */ + +#if !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _KBASE_DEBUG_LINUX_KTRACE_H_ + +#if KBASE_KTRACE_TARGET_FTRACE + +DECLARE_EVENT_CLASS(mali_add_template, + TP_PROTO(u64 info_val), + TP_ARGS(info_val), + TP_STRUCT__entry( + __field(u64, info_val) + ), + TP_fast_assign( + __entry->info_val = info_val; + ), + TP_printk("info=0x%llx", __entry->info_val) +); + +#define DEFINE_MALI_ADD_EVENT(name) \ +DEFINE_EVENT(mali_add_template, mali_##name, \ + TP_PROTO(u64 info_val), \ + TP_ARGS(info_val)) +DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); +DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); +DEFINE_MALI_ADD_EVENT(PM_PWRON); +DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); +DEFINE_MALI_ADD_EVENT(PM_PWROFF); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_GPU_ON); +DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); +DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); +DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CONTEXT_ACTIVE); +DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE); +DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); +DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); +DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); + +#undef DEFINE_MALI_ADD_EVENT + +#include "mali_kbase_debug_linux_ktrace_jm.h" + +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) */ diff --git a/mali_kbase/device/backend/mali_kbase_device_jm.c b/mali_kbase/device/backend/mali_kbase_device_jm.c index 24dbe80..fbba2e7 100644 --- a/mali_kbase/device/backend/mali_kbase_device_jm.c +++ b/mali_kbase/device/backend/mali_kbase_device_jm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. @@ -32,6 +33,10 @@ #include <mali_kbase_model_linux.h> #endif +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include <arbiter/mali_kbase_arbiter_pm.h> +#endif + #include <mali_kbase.h> #include <backend/gpu/mali_kbase_irq_internal.h> #include <backend/gpu/mali_kbase_jm_internal.h> @@ -94,12 +99,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) kbase_pm_context_idle(kbdev); /* Update gpuprops with L2_FEATURES if applicable */ - kbase_gpuprops_update_l2_features(kbdev); + err = kbase_gpuprops_update_l2_features(kbdev); + if (err) + goto fail_update_l2_features; init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); return 0; +fail_update_l2_features: fail_devfreq_init: kbase_job_slot_term(kbdev); fail_job_slot: @@ -146,10 +154,10 @@ static const struct kbase_device_init dev_init[] = { {registers_map, registers_unmap, "Register map failed"}, #endif - {power_control_init, power_control_term, - "Power control initialization failed"}, {kbase_device_io_history_init, kbase_device_io_history_term, "Register access history initialization failed"}, + {kbase_device_pm_init, kbase_device_pm_term, + "Power management initialization failed"}, {kbase_device_early_init, kbase_device_early_term, "Early device initialization failed"}, {kbase_device_populate_max_freq, NULL, diff --git a/mali_kbase/device/mali_kbase_device.c b/mali_kbase/device/mali_kbase_device.c index 4c77929..76f14e5 100644 --- a/mali_kbase/device/mali_kbase_device.c +++ b/mali_kbase/device/mali_kbase_device.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. @@ -50,24 +51,16 @@ #include "backend/gpu/mali_kbase_pm_internal.h" #include "backend/gpu/mali_kbase_irq_internal.h" +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include "arbiter/mali_kbase_arbiter_pm.h" +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + /* NOTE: Magic - 0x45435254 (TRCE in ASCII). * Supports tracing feature provided in the base module. * Please keep it in sync with the value of base module. */ #define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 -#if KBASE_TRACE_ENABLE -static const char *kbasep_trace_code_string[] = { - /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE - * THIS MUST BE USED AT THE START OF THE ARRAY */ -#define KBASE_TRACE_CODE_MAKE_CODE(X) # X -#include "tl/mali_kbase_trace_defs.h" -#undef KBASE_TRACE_CODE_MAKE_CODE -}; -#endif - -#define DEBUG_MESSAGE_SIZE 256 - /* Number of register accesses for the buffer that we allocate during * initialization time. The buffer size can be changed later via debugfs. */ @@ -77,10 +70,6 @@ static DEFINE_MUTEX(kbase_dev_list_lock); static LIST_HEAD(kbase_dev_list); static int kbase_dev_nr; -static int kbasep_trace_init(struct kbase_device *kbdev); -static void kbasep_trace_term(struct kbase_device *kbdev); -static void kbasep_trace_hook_wrapper(void *param); - struct kbase_device *kbase_device_alloc(void) { return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); @@ -176,7 +165,9 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) */ kbase_hw_set_features_mask(kbdev); - kbase_gpuprops_set_features(kbdev); + err = kbase_gpuprops_set_features(kbdev); + if (err) + goto fail; /* On Linux 4.0+, dma coherency is determined from device tree */ #if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) @@ -207,13 +198,13 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) spin_lock_init(&kbdev->hwcnt.lock); - err = kbasep_trace_init(kbdev); + err = kbase_ktrace_init(kbdev); if (err) goto term_as; init_waitqueue_head(&kbdev->cache_clean_wait); - kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); + kbase_debug_assert_register_hook(&kbase_ktrace_hook_wrapper, kbdev); atomic_set(&kbdev->ctx_num, 0); @@ -233,9 +224,11 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) mutex_init(&kbdev->kctx_list_lock); INIT_LIST_HEAD(&kbdev->kctx_list); + spin_lock_init(&kbdev->hwaccess_lock); + return 0; term_trace: - kbasep_trace_term(kbdev); + kbase_ktrace_term(kbdev); term_as: kbase_device_all_as_term(kbdev); as_init_failed: @@ -250,13 +243,13 @@ void kbase_device_misc_term(struct kbase_device *kbdev) WARN_ON(!list_empty(&kbdev->kctx_list)); -#if KBASE_TRACE_ENABLE +#if KBASE_KTRACE_ENABLE kbase_debug_assert_register_hook(NULL, NULL); #endif kbase_instr_backend_term(kbdev); - kbasep_trace_term(kbdev); + kbase_ktrace_term(kbdev); kbase_device_all_as_term(kbdev); } @@ -313,9 +306,8 @@ void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev) int kbase_device_timeline_init(struct kbase_device *kbdev) { - atomic_set(&kbdev->timeline_is_enabled, 0); - return kbase_timeline_init(&kbdev->timeline, - &kbdev->timeline_is_enabled); + atomic_set(&kbdev->timeline_flags, 0); + return kbase_timeline_init(&kbdev->timeline, &kbdev->timeline_flags); } void kbase_device_timeline_term(struct kbase_device *kbdev) @@ -387,274 +379,6 @@ void kbase_device_put_list(const struct list_head *dev_list) } KBASE_EXPORT_TEST_API(kbase_device_put_list); -/* - * Device trace functions - */ -#if KBASE_TRACE_ENABLE - -static int kbasep_trace_init(struct kbase_device *kbdev) -{ - struct kbase_trace *rbuf; - - rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); - - if (!rbuf) - return -EINVAL; - - kbdev->trace_rbuf = rbuf; - spin_lock_init(&kbdev->trace_lock); - return 0; -} - -static void kbasep_trace_term(struct kbase_device *kbdev) -{ - kfree(kbdev->trace_rbuf); -} - -static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) -{ - s32 written = 0; - - /* Initial part of message */ - written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); - - if (trace_msg->katom) - written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); - - written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); - - /* NOTE: Could add function callbacks to handle different message types */ - /* Jobslot present */ - if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) - written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); - - written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); - - /* Refcount present */ - if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) - written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); - - written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); - - /* Rest of message */ - written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); -} - -static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) -{ - char buffer[DEBUG_MESSAGE_SIZE]; - - kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); - dev_dbg(kbdev->dev, "%s", buffer); -} - -void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) -{ - unsigned long irqflags; - struct kbase_trace *trace_msg; - - spin_lock_irqsave(&kbdev->trace_lock, irqflags); - - trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; - - /* Fill the message */ - trace_msg->thread_id = task_pid_nr(current); - trace_msg->cpu = task_cpu(current); - - getnstimeofday(&trace_msg->timestamp); - - trace_msg->code = code; - trace_msg->ctx = ctx; - - if (NULL == katom) { - trace_msg->katom = false; - } else { - trace_msg->katom = true; - trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); - trace_msg->atom_udata[0] = katom->udata.blob[0]; - trace_msg->atom_udata[1] = katom->udata.blob[1]; - } - - trace_msg->gpu_addr = gpu_addr; - trace_msg->jobslot = jobslot; - trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); - trace_msg->info_val = info_val; - trace_msg->flags = flags; - - /* Update the ringbuffer indices */ - kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; - if (kbdev->trace_next_in == kbdev->trace_first_out) - kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; - - /* Done */ - - spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); -} - -void kbasep_trace_clear(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->trace_lock, flags); - kbdev->trace_first_out = kbdev->trace_next_in; - spin_unlock_irqrestore(&kbdev->trace_lock, flags); -} - -void kbasep_trace_dump(struct kbase_device *kbdev) -{ - unsigned long flags; - u32 start; - u32 end; - - dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); - spin_lock_irqsave(&kbdev->trace_lock, flags); - start = kbdev->trace_first_out; - end = kbdev->trace_next_in; - - while (start != end) { - struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; - - kbasep_trace_dump_msg(kbdev, trace_msg); - - start = (start + 1) & KBASE_TRACE_MASK; - } - dev_dbg(kbdev->dev, "TRACE_END"); - - spin_unlock_irqrestore(&kbdev->trace_lock, flags); - - KBASE_TRACE_CLEAR(kbdev); -} - -static void kbasep_trace_hook_wrapper(void *param) -{ - struct kbase_device *kbdev = (struct kbase_device *)param; - - kbasep_trace_dump(kbdev); -} - -#ifdef CONFIG_DEBUG_FS -struct trace_seq_state { - struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; - u32 start; - u32 end; -}; - -static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) -{ - struct trace_seq_state *state = s->private; - int i; - - if (*pos > KBASE_TRACE_SIZE) - return NULL; - i = state->start + *pos; - if ((state->end >= state->start && i >= state->end) || - i >= state->end + KBASE_TRACE_SIZE) - return NULL; - - i &= KBASE_TRACE_MASK; - - return &state->trace_buf[i]; -} - -static void kbasep_trace_seq_stop(struct seq_file *s, void *data) -{ -} - -static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) -{ - struct trace_seq_state *state = s->private; - int i; - - (*pos)++; - - i = (state->start + *pos) & KBASE_TRACE_MASK; - if (i == state->end) - return NULL; - - return &state->trace_buf[i]; -} - -static int kbasep_trace_seq_show(struct seq_file *s, void *data) -{ - struct kbase_trace *trace_msg = data; - char buffer[DEBUG_MESSAGE_SIZE]; - - kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); - seq_printf(s, "%s\n", buffer); - return 0; -} - -static const struct seq_operations kbasep_trace_seq_ops = { - .start = kbasep_trace_seq_start, - .next = kbasep_trace_seq_next, - .stop = kbasep_trace_seq_stop, - .show = kbasep_trace_seq_show, -}; - -static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) -{ - struct kbase_device *kbdev = inode->i_private; - unsigned long flags; - - struct trace_seq_state *state; - - state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); - if (!state) - return -ENOMEM; - - spin_lock_irqsave(&kbdev->trace_lock, flags); - state->start = kbdev->trace_first_out; - state->end = kbdev->trace_next_in; - memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); - spin_unlock_irqrestore(&kbdev->trace_lock, flags); - - return 0; -} - -static const struct file_operations kbasep_trace_debugfs_fops = { - .owner = THIS_MODULE, - .open = kbasep_trace_debugfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release_private, -}; - -void kbasep_trace_debugfs_init(struct kbase_device *kbdev) -{ - debugfs_create_file("mali_trace", S_IRUGO, - kbdev->mali_debugfs_directory, kbdev, - &kbasep_trace_debugfs_fops); -} - -#else -void kbasep_trace_debugfs_init(struct kbase_device *kbdev) -{ -} -#endif /* CONFIG_DEBUG_FS */ - -#else /* KBASE_TRACE_ENABLE */ -static int kbasep_trace_init(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return 0; -} - -static void kbasep_trace_term(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -static void kbasep_trace_hook_wrapper(void *param) -{ - CSTD_UNUSED(param); -} - -void kbasep_trace_dump(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} -#endif /* KBASE_TRACE_ENABLE */ - int kbase_device_early_init(struct kbase_device *kbdev) { int err; @@ -692,7 +416,14 @@ fail_runtime_pm: void kbase_device_early_term(struct kbase_device *kbdev) { +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_release_interrupts(kbdev); + else + kbase_release_interrupts(kbdev); +#else kbase_release_interrupts(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_pm_runtime_term(kbdev); kbasep_platform_device_term(kbdev); } diff --git a/mali_kbase/gpu/mali_kbase_gpu_regmap.h b/mali_kbase/gpu/mali_kbase_gpu_regmap.h index 31abae2..759f30d 100644 --- a/mali_kbase/gpu/mali_kbase_gpu_regmap.h +++ b/mali_kbase/gpu/mali_kbase_gpu_regmap.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -232,9 +232,17 @@ #define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ #define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ +/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. + */ +#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) +#ifdef CONFIG_MALI_DEBUG +#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE) +#else /* CONFIG_MALI_DEBUG */ +#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON) +#endif /* CONFIG_MALI_DEBUG */ + /* * MMU_IRQ_RAWSTAT register values. Values are valid also for * MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. diff --git a/mali_kbase/jm/mali_base_jm_kernel.h b/mali_kbase/jm/mali_base_jm_kernel.h index b61e612..879a436 100644 --- a/mali_kbase/jm/mali_base_jm_kernel.h +++ b/mali_kbase/jm/mali_base_jm_kernel.h @@ -264,7 +264,6 @@ typedef u32 base_context_create_flags; #define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ BASE_TLSTREAM_JOB_DUMPING_ENABLED) - /* * Dependency stuff, keep it private for now. May want to expose it if * we decide to make the number of semaphores a configurable diff --git a/mali_kbase/jm/mali_kbase_jm_defs.h b/mali_kbase/jm/mali_kbase_jm_defs.h index 172217f..aac561b 100644 --- a/mali_kbase/jm/mali_kbase_jm_defs.h +++ b/mali_kbase/jm/mali_kbase_jm_defs.h @@ -30,8 +30,10 @@ #ifndef _KBASE_JM_DEFS_H_ #define _KBASE_JM_DEFS_H_ -/* Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ -#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 +#include "mali_kbase_js_defs.h" + +/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ +#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 /* * Number of milliseconds before resetting the GPU when a job cannot be "zapped" @@ -770,6 +772,12 @@ struct kbase_jd_renderpass { * reaching this offset. * @work_id: atomic variable used for GPU tracepoints, * incremented on every call to base_jd_submit. + * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both + * allocate & free, in submission order, protected + * by kbase_jd_context.lock. + * @jit_pending_alloc: A list of just-in-time memory allocation + * soft-jobs which will be reattempted after the + * impending free of other active allocations. */ struct kbase_jd_context { struct mutex lock; @@ -787,6 +795,9 @@ struct kbase_jd_context { #ifdef CONFIG_GPU_TRACEPOINTS atomic_t work_id; #endif + + struct list_head jit_atoms_head; + struct list_head jit_pending_alloc; }; /** diff --git a/mali_kbase/jm/mali_kbase_jm_ioctl.h b/mali_kbase/jm/mali_kbase_jm_ioctl.h index 127d990..408e98e 100644 --- a/mali_kbase/jm/mali_kbase_jm_ioctl.h +++ b/mali_kbase/jm/mali_kbase_jm_ioctl.h @@ -92,9 +92,11 @@ * flags member. Previous variants of this structure are kept and given _10_2 * and _11_5 suffixes. * - The above changes are checked for safe values in usual builds + * 11.21: + * - v2.0 of mali_trace debugfs file, which now versions the file separately */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 20 +#define BASE_UK_VERSION_MINOR 21 /** * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel diff --git a/mali_kbase/mali_kbase_js_defs.h b/mali_kbase/jm/mali_kbase_js_defs.h index f858687..0b48615 100644 --- a/mali_kbase/mali_kbase_js_defs.h +++ b/mali_kbase/jm/mali_kbase_js_defs.h @@ -21,7 +21,6 @@ */ - /** * @file mali_kbase_js.h * Job Scheduler Type Definitions diff --git a/mali_kbase/mali_base_hwconfig_issues.h b/mali_kbase/mali_base_hwconfig_issues.h index 94c89fa..3966069 100644 --- a/mali_kbase/mali_base_hwconfig_issues.h +++ b/mali_kbase/mali_base_hwconfig_issues.h @@ -58,6 +58,7 @@ enum base_hw_issue { BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_END }; @@ -349,6 +350,7 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_END }; @@ -364,6 +366,7 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_END }; @@ -406,6 +409,7 @@ static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_END }; @@ -446,6 +450,7 @@ static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_END }; @@ -500,6 +505,33 @@ static const enum base_hw_issue base_hw_issues_model_tBEx[] = { BASE_HW_ISSUE_END }; +static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, diff --git a/mali_kbase/mali_kbase.h b/mali_kbase/mali_kbase.h index 66e4349..0445e0c 100644 --- a/mali_kbase/mali_kbase.h +++ b/mali_kbase/mali_kbase.h @@ -56,10 +56,10 @@ */ #include "mali_kbase_defs.h" +#include "debug/mali_kbase_debug_ktrace.h" #include "context/mali_kbase_context.h" #include "mali_kbase_strings.h" #include "mali_kbase_mem_lowlevel.h" -#include "mali_kbase_js.h" #include "mali_kbase_utility.h" #include "mali_kbase_mem.h" #include "mmu/mali_kbase_mmu.h" @@ -70,6 +70,7 @@ #include "mali_kbase_debug_job_fault.h" #include "mali_kbase_jd_debugfs.h" #include "mali_kbase_jm.h" +#include "mali_kbase_js.h" #include "ipa/mali_kbase_ipa.h" @@ -161,6 +162,25 @@ void kbase_sysfs_term(struct kbase_device *kbdev); int kbase_protected_mode_init(struct kbase_device *kbdev); void kbase_protected_mode_term(struct kbase_device *kbdev); +/** + * kbase_device_pm_init() - Performs power management initialization and + * Verifies device tree configurations. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if successful, otherwise a standard Linux error code + */ +int kbase_device_pm_init(struct kbase_device *kbdev); + +/** + * kbase_device_pm_term() - Performs power management deinitialization and + * Free resources. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Clean up all the resources + */ +void kbase_device_pm_term(struct kbase_device *kbdev); + + int power_control_init(struct kbase_device *kbdev); void power_control_term(struct kbase_device *kbdev); @@ -385,6 +405,24 @@ static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) return kbdev->pm.suspending; } +#ifdef CONFIG_MALI_ARBITER_SUPPORT +/* + * Check whether a gpu lost is in progress + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Indicates whether a gpu lost has been received and jobs are no longer + * being scheduled + * + * Return: false if gpu is lost + * Return: != false otherwise + */ +static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) +{ + return kbdev->pm.gpu_lost; +} +#endif + /** * kbase_pm_is_active - Determine whether the GPU is active * @@ -534,181 +572,6 @@ void kbase_disjoint_state_down(struct kbase_device *kbdev); #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) #endif -#if KBASE_TRACE_ENABLE -void kbasep_trace_debugfs_init(struct kbase_device *kbdev); - -#ifndef CONFIG_MALI_SYSTEM_TRACE -/** Add trace values about a job-slot - * - * @note Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any - * functions called to get the parameters supplied to this macro must: - * - be static or static inline - * - must just return 0 and have no other statements present in the body. - */ -#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ - kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ - KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) - -/** Add trace values about a job-slot, with info - * - * @note Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any - * functions called to get the parameters supplied to this macro must: - * - be static or static inline - * - must just return 0 and have no other statements present in the body. - */ -#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ - kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ - KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) - -/** Add trace values about a ctx refcount - * - * @note Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any - * functions called to get the parameters supplied to this macro must: - * - be static or static inline - * - must just return 0 and have no other statements present in the body. - */ -#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ - kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ - KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) -/** Add trace values about a ctx refcount, and info - * - * @note Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any - * functions called to get the parameters supplied to this macro must: - * - be static or static inline - * - must just return 0 and have no other statements present in the body. - */ -#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ - kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ - KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) - -/** Add trace values (no slot or refcount) - * - * @note Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any - * functions called to get the parameters supplied to this macro must: - * - be static or static inline - * - must just return 0 and have no other statements present in the body. - */ -#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ - kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ - 0, 0, 0, info_val) - -/** Clear the trace */ -#define KBASE_TRACE_CLEAR(kbdev) \ - kbasep_trace_clear(kbdev) - -/** Dump the slot trace */ -#define KBASE_TRACE_DUMP(kbdev) \ - kbasep_trace_dump(kbdev) - -/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ -void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); -/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ -void kbasep_trace_clear(struct kbase_device *kbdev); -#else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ -/* Dispatch kbase trace events as system trace events */ -#include <mali_linux_kbase_trace.h> -#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ - trace_mali_##code(jobslot, 0) - -#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ - trace_mali_##code(jobslot, info_val) - -#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ - trace_mali_##code(refcount, 0) - -#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ - trace_mali_##code(refcount, info_val) - -#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ - trace_mali_##code(gpu_addr, info_val) - -#define KBASE_TRACE_CLEAR(kbdev)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(0);\ - } while (0) -#define KBASE_TRACE_DUMP(kbdev)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(0);\ - } while (0) - -#endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ -#else -#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(ctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(gpu_addr);\ - CSTD_UNUSED(jobslot);\ - } while (0) - -#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(ctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(gpu_addr);\ - CSTD_UNUSED(jobslot);\ - CSTD_UNUSED(info_val);\ - CSTD_NOP(0);\ - } while (0) - -#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(ctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(gpu_addr);\ - CSTD_UNUSED(refcount);\ - CSTD_NOP(0);\ - } while (0) - -#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(ctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(gpu_addr);\ - CSTD_UNUSED(info_val);\ - CSTD_NOP(0);\ - } while (0) - -#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(subcode);\ - CSTD_UNUSED(ctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(val);\ - CSTD_NOP(0);\ - } while (0) - -#define KBASE_TRACE_CLEAR(kbdev)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(0);\ - } while (0) -#define KBASE_TRACE_DUMP(kbdev)\ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(0);\ - } while (0) -#endif /* KBASE_TRACE_ENABLE */ -/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ -void kbasep_trace_dump(struct kbase_device *kbdev); - #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) /* kbase_io_history_init - initialize data struct for register access history @@ -758,5 +621,4 @@ int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); #endif /* CONFIG_DEBUG_FS */ - #endif diff --git a/mali_kbase/mali_kbase_core_linux.c b/mali_kbase/mali_kbase_core_linux.c index 3f3d5cc..fb2353e 100644 --- a/mali_kbase/mali_kbase_core_linux.c +++ b/mali_kbase/mali_kbase_core_linux.c @@ -59,12 +59,16 @@ #include "mali_kbase_hwcnt_virtualizer.h" #include "mali_kbase_hwcnt_legacy.h" #include "mali_kbase_vinstr.h" +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include "arbiter/mali_kbase_arbiter_pm.h" +#endif #include "mali_kbase_cs_experimental.h" #ifdef CONFIG_MALI_CINSTR_GWT #include "mali_kbase_gwt.h" #endif +#include "mali_kbase_pm_internal.h" #include <linux/module.h> #include <linux/init.h> @@ -852,7 +856,7 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo) { u32 flags = timeinfo->in.request_flags; - struct timespec ts; + struct timespec64 ts; u64 timestamp; u64 cycle_cnt; @@ -2432,11 +2436,16 @@ struct kbasep_debug_command { kbasep_debug_command_func *func; }; +void kbasep_ktrace_dump_wrapper(struct kbase_device *kbdev) +{ + KBASE_KTRACE_DUMP(kbdev); +} + /* Debug commands supported by the driver */ static const struct kbasep_debug_command debug_commands[] = { { .str = "dumptrace", - .func = &kbasep_trace_dump, + .func = &kbasep_ktrace_dump_wrapper, } }; @@ -2558,9 +2567,9 @@ static ssize_t kbase_show_gpuinfo(struct device *dev, { .id = GPU_ID2_PRODUCT_TTRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, .name = "Mali-G77" }, { .id = GPU_ID2_PRODUCT_TBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, - .name = "Mali-TBEX" }, + .name = "Mali-G78" }, { .id = GPU_ID2_PRODUCT_LBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, - .name = "Mali-LBEX" }, + .name = "Mali-G68" }, { .id = GPU_ID2_PRODUCT_TNAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, .name = "Mali-G57" }, { .id = GPU_ID2_PRODUCT_TODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, @@ -3373,6 +3382,124 @@ void registers_unmap(struct kbase_device *kbdev) kbase_common_reg_unmap(kbdev); } +#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) + +static bool kbase_is_pm_enabled(const struct device_node *gpu_node) +{ + const struct device_node *power_model_node; + const void *cooling_cells_node; + const void *operating_point_node; + bool is_pm_enable = false; + + power_model_node = of_get_child_by_name(gpu_node, + "power_model"); + if (power_model_node) + is_pm_enable = true; + + cooling_cells_node = of_get_property(gpu_node, + "#cooling-cells", NULL); + if (cooling_cells_node) + is_pm_enable = true; + + operating_point_node = of_get_property(gpu_node, + "operating-points", NULL); + if (operating_point_node) + is_pm_enable = true; + + return is_pm_enable; +} + +static bool kbase_is_pv_enabled(const struct device_node *gpu_node) +{ + const void *arbiter_if_node; + + arbiter_if_node = of_get_property(gpu_node, + "arbiter_if", NULL); + + return arbiter_if_node ? true : false; +} + +static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) +{ + const void *coherency_dts; + u32 coherency; + + coherency_dts = of_get_property(gpu_node, + "system-coherency", + NULL); + if (coherency_dts) { + coherency = be32_to_cpup(coherency_dts); + if (coherency == COHERENCY_ACE) + return true; + } + return false; +} + +#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ + +int kbase_device_pm_init(struct kbase_device *kbdev) +{ + int err = 0; + +#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) + + u32 gpu_id; + u32 product_id; + u32 gpu_model_id; + + if (kbase_is_pv_enabled(kbdev->dev->of_node)) { + if (kbase_is_pm_enabled(kbdev->dev->of_node)) { + /* Arbitration AND power management invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); + return -EPERM; + } + if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { + /* Arbitration AND full coherency invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); + return -EPERM; + } + err = kbase_arbiter_pm_early_init(kbdev); + if (err == 0) { + /* Check if Arbitration is running on + * supported GPU platform + */ + kbase_pm_register_access_enable(kbdev); + gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + kbase_pm_register_access_disable(kbdev); + product_id = KBASE_UBFX32(gpu_id, + GPU_ID_VERSION_PRODUCT_ID_SHIFT, 16); + gpu_model_id = GPU_ID2_MODEL_MATCH_VALUE(product_id); + + if (gpu_model_id != GPU_ID2_PRODUCT_TGOX + && gpu_model_id != GPU_ID2_PRODUCT_TNOX) { + kbase_arbiter_pm_early_term(kbdev); + dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); + return -EPERM; + } + } + } else { + err = power_control_init(kbdev); + } +#else + err = power_control_init(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ + return err; +} + +void kbase_device_pm_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#ifdef CONFIG_OF + if (kbase_is_pv_enabled(kbdev->dev->of_node)) + kbase_arbiter_pm_early_term(kbdev); + else + power_control_term(kbdev); +#endif /* CONFIG_OF */ +#else + power_control_term(kbdev); +#endif +} + int power_control_init(struct kbase_device *kbdev) { #if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) @@ -3770,9 +3897,7 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) kbdev->mali_debugfs_directory, kbdev, &fops_trigger_reset); -#if KBASE_TRACE_ENABLE - kbasep_trace_debugfs_init(kbdev); -#endif /* KBASE_TRACE_ENABLE */ + kbase_ktrace_debugfs_init(kbdev); #ifdef CONFIG_MALI_DEVFREQ #ifdef CONFIG_DEVFREQ_THERMAL @@ -4006,6 +4131,11 @@ static int kbase_platform_device_probe(struct platform_device *pdev) "Probed as %s\n", dev_name(kbdev->mdev.this_device)); #endif /* MALI_KBASE_BUILD */ kbase_increment_device_id(); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + mutex_lock(&kbdev->pm.lock); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); + mutex_unlock(&kbdev->pm.lock); +#endif } return err; @@ -4248,7 +4378,6 @@ MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ __stringify(BASE_UK_VERSION_MAJOR) "." \ __stringify(BASE_UK_VERSION_MINOR) ")"); - #define CREATE_TRACE_POINTS /* Create the trace points (otherwise we just get code to call a tracepoint) */ #include "mali_linux_trace.h" @@ -4282,6 +4411,3 @@ void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event) trace_mali_total_alloc_pages_change(dev_id, event); } #endif /* CONFIG_MALI_GATOR_SUPPORT */ -#ifdef CONFIG_MALI_SYSTEM_TRACE -#include "mali_linux_kbase_trace.h" -#endif diff --git a/mali_kbase/mali_kbase_ctx_sched.c b/mali_kbase/mali_kbase_ctx_sched.c index 3922260..cea91bc 100644 --- a/mali_kbase/mali_kbase_ctx_sched.c +++ b/mali_kbase/mali_kbase_ctx_sched.c @@ -23,7 +23,23 @@ #include <mali_kbase.h> #include <mali_kbase_config_defaults.h> +#include <mali_kbase_defs.h> #include "mali_kbase_ctx_sched.h" +#include "tl/mali_kbase_tracepoints.h" + +/* Helper for ktrace */ +#if KBASE_KTRACE_ENABLE +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} +#else /* KBASE_KTRACE_ENABLE */ +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + CSTD_UNUSED(kctx); + return 0; +} +#endif /* KBASE_KTRACE_ENABLE */ int kbase_ctx_sched_init(struct kbase_device *kbdev) { @@ -106,11 +122,15 @@ int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) if (prev_kctx) { WARN_ON(atomic_read(&prev_kctx->refcount) != 0); kbase_mmu_disable(prev_kctx); + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, prev_kctx->id); prev_kctx->as_nr = KBASEP_AS_NR_INVALID; } kctx->as_nr = free_as; kbdev->as_to_kctx[free_as] = kctx; + KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( + kbdev, kctx->id, free_as); kbase_mmu_update(kbdev, &kctx->mmu, kctx->as_nr); } @@ -142,17 +162,23 @@ void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) { struct kbase_device *const kbdev = kctx->kbdev; + int new_ref_count; lockdep_assert_held(&kbdev->hwaccess_lock); - if (atomic_dec_return(&kctx->refcount) == 0) { + new_ref_count = atomic_dec_return(&kctx->refcount); + if (new_ref_count == 0) { kbdev->as_free |= (1u << kctx->as_nr); if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) { + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, kctx->id); kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT); } } + + KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count); } void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) @@ -168,6 +194,7 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) if (kbdev->pm.backend.gpu_powered) kbase_mmu_disable(kctx); + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(kbdev, kctx->id); kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; } @@ -198,11 +225,120 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) /* This context might have been assigned an * AS before, clear it. */ - kbdev->as_to_kctx[kctx->as_nr] = NULL; - kctx->as_nr = KBASEP_AS_NR_INVALID; + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, kctx->id); + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } } } else { kbase_mmu_disable_as(kbdev, i); } } } + +struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( + struct kbase_device *kbdev, size_t as_nr) +{ + unsigned long flags; + struct kbase_context *found_kctx = NULL; + + if (WARN_ON(kbdev == NULL)) + return NULL; + + if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) + return NULL; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (found_kctx != NULL) + kbase_ctx_sched_retain_ctx_refcount(found_kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, + size_t as_nr) +{ + struct kbase_context *found_kctx; + + if (WARN_ON(kbdev == NULL)) + return NULL; + + if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) + return NULL; + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (WARN_ON(!found_kctx)) + return NULL; + + if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0)) + return NULL; + + return found_kctx; +} + +bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx) +{ + bool result = false; + int as_nr; + + if (WARN_ON(kctx == NULL)) + return result; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + as_nr = kctx->as_nr; + if (atomic_read(&kctx->refcount) > 0) { + KBASE_DEBUG_ASSERT(as_nr >= 0); + + kbase_ctx_sched_retain_ctx_refcount(kctx); + KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, + kbase_ktrace_get_ctx_refcnt(kctx)); + result = true; + } + + return result; +} + +bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx) +{ + unsigned long flags; + bool result = false; + + if (WARN_ON(kctx == NULL)) + return result; + + if (WARN_ON(kctx->kbdev == NULL)) + return result; + + mutex_lock(&kctx->kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + result = kbase_ctx_sched_inc_refcount_nolock(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->kbdev->mmu_hw_mutex); + + return result; +} + +void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) +{ + unsigned long flags; + + if (WARN_ON(!kctx)) + return; + + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + + if (!WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID) && + !WARN_ON(atomic_read(&kctx->refcount) <= 0)) + kbase_ctx_sched_release_ctx(kctx); + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); +} diff --git a/mali_kbase/mali_kbase_ctx_sched.h b/mali_kbase/mali_kbase_ctx_sched.h index ab57a0d..1affa71 100644 --- a/mali_kbase/mali_kbase_ctx_sched.h +++ b/mali_kbase/mali_kbase_ctx_sched.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -132,4 +132,78 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); */ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); +/** + * kbase_ctx_sched_as_to_ctx_refcount - Lookup a context based on its current + * address space and ensure that is stays scheduled in + * @kbdev: The device for which the returned context must belong + * @as_nr: address space assigned to the context of interest + * + * The context is refcounted as being busy to prevent it from scheduling + * out. It must be released with kbase_ctx_sched_release_ctx() when it is no + * longer required to stay scheduled in. + * + * This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * * it must not hold the kbase_device::hwaccess_lock, because it will be used + * internally. + * + * Return: a valid struct kbase_context on success, which has been refcounted + * as being busy or return NULL on failure, indicating that no context was found + * in as_nr. + */ +struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( + struct kbase_device *kbdev, size_t as_nr); + +/** + * kbase_ctx_sched_as_to_ctx - Lookup a context based on its current address + * space + * @kbdev: The device for which the returned context must belong + * @as_nr: address space assigned to the context of interest + * + * Return: a valid struct kbase_context on success or NULL on failure, + * indicating that no context was found in as_nr. + */ +struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, + size_t as_nr); + +/** + * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy, + * preventing it from being scheduled out. + * @kctx: Context to be refcounted + * + * The following locks must be held by the caller: + * * kbase_device::mmu_hw_mutex + * * kbase_device::hwaccess_lock + * + * Return: true if refcount succeeded, and the context will not be scheduled + * out, false if the refcount failed (because the context is being/has been + * scheduled out). + */ +bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_inc_refcount - Refcount a context as being busy, preventing + * it from being scheduled out. + * @kctx: Context to be refcounted + * + * The following locking conditions are made on the caller: + * * it must not hold kbase_device::mmu_hw_mutex and + * kbase_device::hwaccess_lock, because they will be used internally. + * + * Return: true if refcount succeeded, and the context will not be scheduled + * out, false if the refcount failed (because the context is being/has been + * scheduled out). + */ +bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_release_ctx_lock - Release a reference count of a context + * @kctx: Context for which refcount should be decreased + * + * Effectivelly, this is a wrapper for kbase_ctx_sched_release_ctx, but + * kbase_device::hwaccess_lock is required NOT to be locked. + */ +void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); + #endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/mali_kbase/mali_kbase_defs.h b/mali_kbase/mali_kbase_defs.h index ce32b53..7056d80 100644 --- a/mali_kbase/mali_kbase_defs.h +++ b/mali_kbase/mali_kbase_defs.h @@ -61,12 +61,16 @@ #ifdef CONFIG_DEBUG_FS #include <linux/debugfs.h> -#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_DEBUG_FS */ #ifdef CONFIG_MALI_DEVFREQ #include <linux/devfreq.h> #endif /* CONFIG_MALI_DEVFREQ */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include <arbiter/mali_kbase_arbiter_defs.h> +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + #include <linux/clk.h> #include <linux/regulator/consumer.h> #include <linux/memory_group_manager.h> @@ -76,18 +80,7 @@ #define KBASE_PM_RUNTIME 1 #endif -/** Enable SW tracing when set */ -#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE -#define KBASE_TRACE_ENABLE 1 -#endif - -#ifndef KBASE_TRACE_ENABLE -#ifdef CONFIG_MALI_DEBUG -#define KBASE_TRACE_ENABLE 1 -#else -#define KBASE_TRACE_ENABLE 0 -#endif /* CONFIG_MALI_DEBUG */ -#endif /* KBASE_TRACE_ENABLE */ +#include "debug/mali_kbase_debug_ktrace_defs.h" /** Number of milliseconds before we time out on a GPU soft/hard reset */ #define RESET_TIMEOUT 500 @@ -130,11 +123,6 @@ */ #define KBASE_LOCK_REGION_MIN_SIZE_LOG2 (15) -#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ -#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) -#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) - -#include "mali_kbase_js_defs.h" #include "mali_kbase_hwaccess_defs.h" /* Maximum number of pages of memory that require a permanent mapping, per @@ -332,70 +320,6 @@ struct kbasep_mem_device { atomic_t ir_threshold; }; -#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X - -enum kbase_trace_code { - /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE - * THIS MUST BE USED AT THE START OF THE ENUM */ -#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) -#include <tl/mali_kbase_trace_defs.h> -#undef KBASE_TRACE_CODE_MAKE_CODE - /* Comma on its own, to extend the list */ - , - /* Must be the last in the enum */ - KBASE_TRACE_CODE_COUNT -}; - -#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) -#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) - -/** - * struct kbase_trace - object representing a trace message added to trace buffer - * kbase_device::trace_rbuf - * @timestamp: CPU timestamp at which the trace message was added. - * @thread_id: id of the thread in the context of which trace message - * was added. - * @cpu: indicates which CPU the @thread_id was scheduled on when - * the trace message was added. - * @ctx: Pointer to the kbase context for which the trace message - * was added. Will be NULL for certain trace messages like - * for traces added corresponding to power management events. - * Will point to the appropriate context corresponding to - * job-slot & context's reference count related events. - * @katom: indicates if the trace message has atom related info. - * @atom_number: id of the atom for which trace message was added. - * Only valid if @katom is true. - * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. - * Only valid if @katom is true. - * @gpu_addr: GPU address of the job-chain represented by atom. Could - * be valid even if @katom is false. - * @info_val: value specific to the type of event being traced. For the - * case where @katom is true, will be set to atom's affinity, - * i.e. bitmask of shader cores chosen for atom's execution. - * @code: Identifies the event, refer enum kbase_trace_code. - * @jobslot: job-slot for which trace message was added, valid only for - * job-slot management events. - * @refcount: reference count for the context, valid for certain events - * related to scheduler core and policy. - * @flags: indicates if info related to @jobslot & @refcount is present - * in the trace message, used during dumping of the message. - */ -struct kbase_trace { - struct timespec timestamp; - u32 thread_id; - u32 cpu; - void *ctx; - bool katom; - int atom_number; - u64 atom_udata[2]; - u64 gpu_addr; - unsigned long info_val; - u8 code; - u8 jobslot; - u8 refcount; - u8 flags; -}; - /** * Data stored per device for power management. * @@ -420,6 +344,10 @@ struct kbase_pm_device_data { int active_count; /** Flag indicating suspending/suspended */ bool suspending; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* Flag indicating gpu lost */ + bool gpu_lost; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* Wait queue set when active_count == 0 */ wait_queue_head_t zero_active_count_wait; @@ -450,6 +378,13 @@ struct kbase_pm_device_data { u32 dvfs_period; struct kbase_pm_backend_data backend; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /** + * The state of the arbiter VM machine + */ + struct kbase_arbiter_vm_state *arb_vm_state; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ }; /** @@ -722,8 +657,9 @@ struct kbase_devfreq_queue_info { * kbase_hwcnt_context_enable() with @hwcnt_gpu_ctx. * @hwcnt_gpu_virt: Virtualizer for GPU hardware counters. * @vinstr_ctx: vinstr context created per device. - * @timeline_is_enabled: Non zero, if there is at least one timeline client, - * zero otherwise. + * @timeline_flags: Bitmask defining which sets of timeline tracepoints + * are enabled. If zero, there is no timeline client and + * therefore timeline is disabled. * @timeline: Timeline context created per device. * @trace_lock: Lock to serialize the access to trace buffer. * @trace_first_out: Index/offset in the trace buffer at which the first @@ -913,7 +849,6 @@ struct kbase_device { struct kbase_pm_device_data pm; - struct kbasep_js_device_data js_data; struct kbase_mem_pool_group mem_pools; struct kbasep_mem_device memdev; struct kbase_mmu_mode const *mmu_mode; @@ -955,14 +890,11 @@ struct kbase_device { struct kbase_hwcnt_virtualizer *hwcnt_gpu_virt; struct kbase_vinstr_context *vinstr_ctx; - atomic_t timeline_is_enabled; + atomic_t timeline_flags; struct kbase_timeline *timeline; -#if KBASE_TRACE_ENABLE - spinlock_t trace_lock; - u16 trace_first_out; - u16 trace_next_in; - struct kbase_trace *trace_rbuf; +#if KBASE_KTRACE_TARGET_RBUF + struct kbase_ktrace ktrace; #endif u32 reset_timeout_ms; @@ -1098,6 +1030,8 @@ struct kbase_device { u8 l2_size_override; u8 l2_hash_override; + struct kbasep_js_device_data js_data; + /* See KBASE_JS_*_PRIORITY_MODE for details. */ u32 js_ctx_scheduling_mode; @@ -1115,6 +1049,11 @@ struct kbase_device { int slot; u64 flags; } dummy_job_wa; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* Pointer to the arbiter device */ + struct kbase_arbiter_device arb; +#endif }; #define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ @@ -1478,12 +1417,6 @@ struct kbase_sub_alloc { * @jit_work: Work item queued to defer the freeing of a memory * region when a just-in-time memory allocation is moved * to @jit_destroy_head. - * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both - * allocate & free, in submission order, protected by - * &struct_kbase_jd_context.lock. - * @jit_pending_alloc: A list of just-in-time memory allocation soft-jobs - * which will be reattempted after the impending free of - * other active allocations. * @ext_res_meta_head: A list of sticky external resources which were requested to * be mapped on GPU side, through a softjob atom of type * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. @@ -1500,7 +1433,7 @@ struct kbase_sub_alloc { * @gwt_snapshot_list: Snapshot of the @gwt_current_list for sending to user space. * @priority: Indicates the context priority. Used along with @atoms_count * for context scheduling, protected by hwaccess_lock. - * @atoms_count: Number of gpu atoms currently in use, per priority + * @atoms_count: Number of GPU atoms currently in use, per priority * @create_flags: Flags used in context creation. * * A kernel base context is an entity among which the GPU is scheduled. @@ -1563,7 +1496,7 @@ struct kbase_context { pid_t tgid; pid_t pid; atomic_t used_pages; - atomic_t nonmapped_pages; + atomic_t nonmapped_pages; atomic_t permanent_mapped_pages; struct kbase_mem_pool_group mem_pools; @@ -1622,9 +1555,6 @@ struct kbase_context { struct mutex jit_evict_lock; struct work_struct jit_work; - struct list_head jit_atoms_head; - struct list_head jit_pending_alloc; - struct list_head ext_res_meta_head; u8 trim_level; diff --git a/mali_kbase/mali_kbase_dummy_job_wa.c b/mali_kbase/mali_kbase_dummy_job_wa.c index 5830e8e..188e53b 100644 --- a/mali_kbase/mali_kbase_dummy_job_wa.c +++ b/mali_kbase/mali_kbase_dummy_job_wa.c @@ -257,74 +257,15 @@ static ssize_t show_dummy_job_wa_info(struct device * const dev, static DEVICE_ATTR(dummy_job_wa_info, 0444, show_dummy_job_wa_info, NULL); -#define FAIL_PROBE 1 -#define SKIP_WA 2 -#define LOAD_WA 3 - -static int check_wa_validity(struct kbase_device *kbdev, - bool wa_blob_present) +static bool wa_blob_load_needed(struct kbase_device *kbdev) { - struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; - const u32 major_revision = gpu_props->core_props.major_revision; - const u32 minor_revision = gpu_props->core_props.minor_revision; - const u32 gpu_id = gpu_props->raw_props.gpu_id; - const u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> - GPU_ID_VERSION_PRODUCT_ID_SHIFT; - int ret = FAIL_PROBE; - - if (IS_ENABLED(CONFIG_ARCH_VEXPRESS)) - return SKIP_WA; - - switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { - case GPU_ID2_PRODUCT_TTRX: - /* WA needed for r0p0, r0p1 only */ - if (major_revision == 0) { - if ((minor_revision <= 1) && wa_blob_present) - ret = LOAD_WA; - else if ((minor_revision > 1) && !wa_blob_present) - ret = SKIP_WA; - } else if ((major_revision > 0) && !wa_blob_present) - ret = SKIP_WA; - break; - case GPU_ID2_PRODUCT_TNAX: - /* WA needed for r0p0 only */ - if (major_revision == 0) { - if ((minor_revision == 0) && wa_blob_present) - ret = LOAD_WA; - else if ((minor_revision > 0) && !wa_blob_present) - ret = SKIP_WA; - } else if ((major_revision > 0) && !wa_blob_present) - ret = SKIP_WA; - break; - case GPU_ID2_PRODUCT_TBEX: - /* WA needed for r0p0 only */ - if ((major_revision == 0) && (minor_revision == 0)) { - if (!wa_blob_present) { - dev_warn(kbdev->dev, "Dummy job WA not applied, susceptible to GPU hang. Contact support-mali@arm.com"); - ret = SKIP_WA; - } else - ret = LOAD_WA; - } else if (!wa_blob_present) - ret = SKIP_WA; - break; - case GPU_ID2_PRODUCT_LBEX: - /* WA needed for r1p0 only */ - if ((major_revision == 1) && (minor_revision == 0)) { - if (!wa_blob_present) { - dev_warn(kbdev->dev, "Dummy job WA not applied, susceptible to GPU hang. Contact support-mali@arm.com"); - ret = SKIP_WA; - } else - ret = LOAD_WA; - } else if (!wa_blob_present) - ret = SKIP_WA; - break; - default: - if (!wa_blob_present) - ret = SKIP_WA; - break; - } + if (of_machine_is_compatible("arm,juno")) + return false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485)) + return true; - return ret; + return false; } int kbase_dummy_job_wa_load(struct kbase_device *kbdev) @@ -339,31 +280,17 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) const struct wa_v2_info *v2_info; u32 blob_offset; int err; - int ret; struct kbase_context *kctx; + if (!wa_blob_load_needed(kbdev)) + return 0; + /* load the wa */ -#if KERNEL_VERSION(4, 18, 0) <= LINUX_VERSION_CODE - err = firmware_request_nowarn(&firmware, wa_name, kbdev->dev); -#else err = request_firmware(&firmware, wa_name, kbdev->dev); -#endif - ret = check_wa_validity(kbdev, err == 0); - - if (ret == SKIP_WA) { - if (err == 0) - release_firmware(firmware); - return 0; - } else if (ret == FAIL_PROBE) { - if (err == 0) { - dev_err(kbdev->dev, "WA blob unexpectedly present. Please refer to the Arm Mali DDK Bifrost/Valhall Release Notes, " - "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); - release_firmware(firmware); - } else { - dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " - "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); - } + if (err) { + dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " + "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); return -ENODEV; } @@ -504,8 +431,9 @@ void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); wa_ctx = READ_ONCE(kbdev->dummy_job_wa.ctx); + WRITE_ONCE(kbdev->dummy_job_wa.ctx, NULL); /* make this write visible before we tear down the ctx */ - smp_store_mb(kbdev->dummy_job_wa.ctx, NULL); + smp_mb(); if (wa_ctx) { kbasep_js_release_privileged_ctx(kbdev, wa_ctx); diff --git a/mali_kbase/mali_kbase_event.c b/mali_kbase/mali_kbase_event.c index 2bbc313..c8b8f22 100644 --- a/mali_kbase/mali_kbase_event.c +++ b/mali_kbase/mali_kbase_event.c @@ -25,6 +25,7 @@ #include <mali_kbase.h> #include <mali_kbase_debug.h> #include <tl/mali_kbase_tracepoints.h> +#include <mali_linux_trace.h> static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) { @@ -200,6 +201,10 @@ void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) dev_dbg(kbdev->dev, "Reporting %d events\n", event_count); kbase_event_wakeup(ctx); + + /* Post-completion latency */ + trace_sysgraph(SGR_POST, ctx->id, + kbase_jd_atom_id(ctx, atom)); } } KBASE_EXPORT_TEST_API(kbase_event_post); diff --git a/mali_kbase/mali_kbase_gpuprops.c b/mali_kbase/mali_kbase_gpuprops.c index f1f188f..ae2458f 100644 --- a/mali_kbase/mali_kbase_gpuprops.c +++ b/mali_kbase/mali_kbase_gpuprops.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. @@ -37,19 +38,6 @@ #include <linux/of_platform.h> #include <linux/moduleparam.h> -/** - * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. - * @value: The value from which to extract bits. - * @offset: The first bit to extract (0 being the LSB). - * @size: The number of bits to extract. - * - * Context: @offset + @size <= 32. - * - * Return: Bits [@offset, @offset + @size) from @value. - */ -/* from mali_cdsb.h */ -#define KBASE_UBFX32(value, offset, size) \ - (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) static void kbase_gpuprops_construct_coherent_groups( struct base_gpu_props * const props) @@ -126,18 +114,23 @@ static void kbase_gpuprops_construct_coherent_groups( * * Fill the &struct base_gpu_props structure with values from the GPU * configuration registers. Only the raw properties are filled in this function. + * + * Return: Zero on success, Linux error code on failure */ -static void kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, +static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, struct kbase_device *kbdev) { struct kbase_gpuprops_regdump regdump; int i; + int err; KBASE_DEBUG_ASSERT(NULL != kbdev); KBASE_DEBUG_ASSERT(NULL != gpu_props); /* Dump relevant registers */ - kbase_backend_gpuprops_get(kbdev, ®dump); + err = kbase_backend_gpuprops_get(kbdev, ®dump); + if (err) + return err; gpu_props->raw_props.gpu_id = regdump.gpu_id; gpu_props->raw_props.tiler_features = regdump.tiler_features; @@ -172,6 +165,8 @@ static void kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; gpu_props->raw_props.thread_features = regdump.thread_features; gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc; + + return 0; } void kbase_gpuprops_update_core_props_gpu_id( @@ -325,15 +320,18 @@ void kbase_gpuprops_set(struct kbase_device *kbdev) gpu_props->num_job_slots = hweight32(raw->js_present); } -void kbase_gpuprops_set_features(struct kbase_device *kbdev) +int kbase_gpuprops_set_features(struct kbase_device *kbdev) { struct base_gpu_props *gpu_props; struct kbase_gpuprops_regdump regdump; + int err; gpu_props = &kbdev->gpu_props.props; /* Dump relevant registers */ - kbase_backend_gpuprops_get_features(kbdev, ®dump); + err = kbase_backend_gpuprops_get_features(kbdev, ®dump); + if (err) + return err; /* * Copy the raw value from the register, later this will get turned @@ -345,6 +343,8 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev) if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT)) gpu_props->thread_props.max_thread_group_split = 0; + + return err; } /* @@ -396,15 +396,17 @@ static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) return false; } -void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) +int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) { + int err = 0; + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { struct kbase_gpuprops_regdump regdump; struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; /* Check for L2 cache size & hash overrides */ if (!kbase_read_l2_config_from_dt(kbdev)) - return; + return 0; /* Need L2 to get powered to reflect to L2_FEATURES */ kbase_pm_context_active(kbdev); @@ -413,7 +415,9 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) kbase_pm_wait_for_l2_powered(kbdev); /* Dump L2_FEATURES register */ - kbase_backend_gpuprops_get_l2_features(kbdev, ®dump); + err = kbase_backend_gpuprops_get_l2_features(kbdev, ®dump); + if (err) + goto idle_gpu; dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n", regdump.l2_features); @@ -423,9 +427,12 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); +idle_gpu: /* Let GPU idle */ kbase_pm_context_idle(kbdev); } + + return err; } static struct { diff --git a/mali_kbase/mali_kbase_gpuprops.h b/mali_kbase/mali_kbase_gpuprops.h index eeba92f..5eee794 100644 --- a/mali_kbase/mali_kbase_gpuprops.h +++ b/mali_kbase/mali_kbase_gpuprops.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,20 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * */ @@ -36,6 +50,20 @@ struct kbase_device; /** + * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. + * @value: The value from which to extract bits. + * @offset: The first bit to extract (0 being the LSB). + * @size: The number of bits to extract. + * + * Context: @offset + @size <= 32. + * + * Return: Bits [@offset, @offset + @size) from @value. + */ +/* from mali_cdsb.h */ +#define KBASE_UBFX32(value, offset, size) \ + (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) + +/** * @brief Set up Kbase GPU properties. * * Set up Kbase GPU properties with information from the GPU registers @@ -51,16 +79,20 @@ void kbase_gpuprops_set(struct kbase_device *kbdev); * This function sets up GPU properties that are dependent on the hardware * features bitmask. This function must be preceeded by a call to * kbase_hw_set_features_mask(). + * + * Return: Zero on success, Linux error code on failure */ -void kbase_gpuprops_set_features(struct kbase_device *kbdev); +int kbase_gpuprops_set_features(struct kbase_device *kbdev); /** * kbase_gpuprops_update_l2_features - Update GPU property of L2_FEATURES * @kbdev: Device pointer * * This function updates l2_features and the log2 cache size. + * + * Return: Zero on success, Linux error code for failure */ -void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev); +int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev); /** * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer diff --git a/mali_kbase/mali_kbase_hw.c b/mali_kbase/mali_kbase_hw.c index c3abad4..f8a9248 100644 --- a/mali_kbase/mali_kbase_hw.c +++ b/mali_kbase/mali_kbase_hw.c @@ -97,11 +97,11 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) for (; *features != BASE_HW_FEATURE_END; features++) set_bit(*features, &kbdev->hw_features_mask[0]); -#if defined(CONFIG_MALI_JOB_DUMP) || defined(CONFIG_MALI_VECTOR_DUMP) +#if defined(CONFIG_MALI_VECTOR_DUMP) /* When dumping is enabled, need to disable flush reduction optimization * for GPUs on which it is safe to have only cache clean operation at * the end of job chain. - * This is required to make job dumping work. There is some discrepancy + * This is required to make vector dump work. There is some discrepancy * in the implementation of flush reduction optimization due to * unclear or ambiguous ARCH spec. */ @@ -195,14 +195,14 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( {U32_MAX, NULL} } }, {GPU_ID2_PRODUCT_LBEX, - {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0}, - {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tBEx_r1p1}, + {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1}, {U32_MAX, NULL} } }, {GPU_ID2_PRODUCT_TBEX, {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0}, {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0}, - {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1}, {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0}, {U32_MAX, NULL} } }, diff --git a/mali_kbase/mali_kbase_hwaccess_gpuprops.h b/mali_kbase/mali_kbase_hwaccess_gpuprops.h index 62628b6..3ae0dbe 100644 --- a/mali_kbase/mali_kbase_hwaccess_gpuprops.h +++ b/mali_kbase/mali_kbase_hwaccess_gpuprops.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2015, 2018, 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -18,6 +18,20 @@ * * SPDX-License-Identifier: GPL-2.0 * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2018, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * */ @@ -35,8 +49,10 @@ * @regdump: Pointer to struct kbase_gpuprops_regdump structure * * The caller should ensure that GPU remains powered-on during this function. + * + * Return: Zero for succeess or a Linux error code */ -void kbase_backend_gpuprops_get(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump); /** @@ -47,8 +63,10 @@ void kbase_backend_gpuprops_get(struct kbase_device *kbdev, * * This function reads GPU properties that are dependent on the hardware * features bitmask. It will power-on the GPU if required. + * + * Return: Zero for succeess or a Linux error code */ -void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump); /** @@ -59,8 +77,10 @@ void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, * * This function reads L2_FEATURES register that is dependent on the hardware * features bitmask. It will power-on the GPU if required. + * + * Return: Zero on success, Linux error code on failure */ -void kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, +int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump); diff --git a/mali_kbase/mali_kbase_hwaccess_time.h b/mali_kbase/mali_kbase_hwaccess_time.h index ca0cd79..a61e5b9 100644 --- a/mali_kbase/mali_kbase_hwaccess_time.h +++ b/mali_kbase/mali_kbase_hwaccess_time.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014,2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -37,6 +37,6 @@ * time in */ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, - u64 *system_time, struct timespec *ts); + u64 *system_time, struct timespec64 *ts); #endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/mali_kbase/mali_kbase_hwcnt_backend_gpu.c b/mali_kbase/mali_kbase_hwcnt_backend_gpu.c index ae11630..407c768 100644 --- a/mali_kbase/mali_kbase_hwcnt_backend_gpu.c +++ b/mali_kbase/mali_kbase_hwcnt_backend_gpu.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -73,11 +73,8 @@ struct kbase_hwcnt_backend_gpu { static u64 kbasep_hwcnt_backend_gpu_timestamp_ns( struct kbase_hwcnt_backend *backend) { - struct timespec ts; - (void)backend; - getrawmonotonic(&ts); - return (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + return ktime_get_raw_ns(); } /* GPU backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ diff --git a/mali_kbase/mali_kbase_jd.c b/mali_kbase/mali_kbase_jd.c index b4ae3ba..1a830dd 100644 --- a/mali_kbase/mali_kbase_jd.c +++ b/mali_kbase/mali_kbase_jd.c @@ -34,6 +34,7 @@ #include <mali_kbase_jm.h> #include <mali_kbase_hwaccess_jm.h> #include <tl/mali_kbase_tracepoints.h> +#include <mali_linux_trace.h> #include "mali_kbase_dma_fence.h" #include <mali_kbase_cs_experimental.h> @@ -94,6 +95,8 @@ static bool jd_run_atom(struct kbase_jd_atom *katom) if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { /* Dependency only atom */ + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(katom->kctx, katom)); katom->status = KBASE_JD_ATOM_STATE_COMPLETED; dev_dbg(kctx->kbdev->dev, "Atom %p status to completed\n", (void *)katom); @@ -452,6 +455,9 @@ static inline void jd_resolve_dep(struct list_head *out_list, #endif /* CONFIG_MALI_DMA_FENCE */ if (dep_satisfied) { + trace_sysgraph(SGR_DEP_RES, + dep_atom->kctx->id, + kbase_jd_atom_id(katom->kctx, dep_atom)); dep_atom->in_jd_list = true; list_add_tail(&dep_atom->jd_item, out_list); } @@ -540,6 +546,10 @@ static void jd_try_submitting_deps(struct list_head *out_list, #endif /* CONFIG_MALI_DMA_FENCE */ if (dep0_valid && dep1_valid && dep_satisfied) { + trace_sysgraph(SGR_DEP_RES, + dep_atom->kctx->id, + kbase_jd_atom_id(dep_atom->kctx, + dep_atom)); dep_atom->in_jd_list = true; list_add(&dep_atom->jd_item, out_list); } @@ -576,6 +586,7 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) for (idx = 0; idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx]; idx++) { + enum heap_pointer { LOW = 0, HIGH, COUNT }; size_t size_to_read; u64 read_val; @@ -601,6 +612,8 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) size_to_read = sizeof(*ptr); if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) size_to_read = sizeof(u32); + else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) + size_to_read = sizeof(u64[COUNT]); ptr = kbase_vmap(kctx, reg->heap_info_gpu_addr, size_to_read, &mapping); @@ -618,20 +631,41 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) read_val = READ_ONCE(*(u32 *)ptr); used_pages = PFN_UP(read_val); } else { - u64 addr_end = read_val = READ_ONCE(*ptr); + u64 addr_end; if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - unsigned long extent_bytes = reg->extent << - PAGE_SHIFT; + const unsigned long extent_bytes = reg->extent + << PAGE_SHIFT; + const u64 low_ptr = ptr[LOW]; + const u64 high_ptr = ptr[HIGH]; + + /* As either the low or high pointer could + * consume their partition and move onto the + * next chunk, we need to account for both. + * In the case where nothing has been allocated + * from the high pointer the whole chunk could + * be backed unnecessarily - but the granularity + * is the chunk size anyway and any non-zero + * offset of low pointer from the start of the + * chunk would result in the whole chunk being + * backed. + */ + read_val = max(high_ptr, low_ptr); + /* kbase_check_alloc_sizes() already satisfies * this, but here to avoid future maintenance * hazards */ WARN_ON(!is_power_of_2(extent_bytes)); - addr_end = ALIGN(read_val, extent_bytes); + } else { + addr_end = read_val = READ_ONCE(*ptr); } - used_pages = PFN_UP(addr_end) - reg->start_pfn; + + if (addr_end >= (reg->start_pfn << PAGE_SHIFT)) + used_pages = PFN_UP(addr_end) - reg->start_pfn; + else + used_pages = reg->used_pages; } trace_mali_jit_report(katom, reg, idx, read_val, used_pages); @@ -877,6 +911,8 @@ static bool jd_submit_atom(struct kbase_context *const kctx, katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; katom->softjob_data = NULL; + trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); + #if MALI_JIT_PRESSURE_LIMIT /* Older API version atoms might have random values where jit_id now * lives, but we must maintain backwards compatibility - handle the @@ -1368,7 +1404,7 @@ void kbase_jd_done_worker(struct work_struct *data) dev_dbg(kbdev->dev, "Enter atom %p done worker for kctx %p\n", (void *)katom, (void *)kctx); - KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); kbase_backend_complete_wq(kbdev, katom); @@ -1508,7 +1544,7 @@ void kbase_jd_done_worker(struct work_struct *data) if (context_idle) kbase_pm_context_idle(kbdev); - KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); dev_dbg(kbdev->dev, "Leave atom %p done worker for kctx %p\n", (void *)katom, (void *)kctx); @@ -1546,7 +1582,7 @@ static void jd_cancel_worker(struct work_struct *data) jctx = &kctx->jctx; js_kctx_info = &kctx->jctx.sched_info; - KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); + KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); /* This only gets called on contexts that are scheduled out. Hence, we must * make sure we don't de-ref the number of running jobs (there aren't @@ -1608,7 +1644,7 @@ void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0); kbase_job_check_leave_disjoint(kbdev, katom); @@ -1640,7 +1676,7 @@ void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) KBASE_DEBUG_ASSERT(NULL != kctx); dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom); - KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); + KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); /* This should only be done from a context that is not scheduled */ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); @@ -1664,7 +1700,7 @@ void kbase_jd_zap_context(struct kbase_context *kctx) kbdev = kctx->kbdev; - KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); + KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); kbase_js_zap_context(kctx); diff --git a/mali_kbase/mali_kbase_js.c b/mali_kbase/mali_kbase_js.c index b3ae604..0b0c5bf 100644 --- a/mali_kbase/mali_kbase_js.c +++ b/mali_kbase/mali_kbase_js.c @@ -28,6 +28,7 @@ #include <mali_kbase.h> #include <mali_kbase_js.h> #include <tl/mali_kbase_tracepoints.h> +#include <mali_linux_trace.h> #include <mali_kbase_hw.h> #include <mali_kbase_ctx_sched.h> @@ -37,7 +38,6 @@ #include "mali_kbase_jm.h" #include "mali_kbase_hwaccess_jm.h" - /* * Private types */ @@ -81,22 +81,19 @@ static int kbase_js_get_slot(struct kbase_device *kbdev, static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_ctx_job_cb callback); -/* Helper for trace subcodes */ -#if KBASE_TRACE_ENABLE -static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, - struct kbase_context *kctx) +/* Helper for ktrace */ +#if KBASE_KTRACE_ENABLE +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) { return atomic_read(&kctx->refcount); } -#else /* KBASE_TRACE_ENABLE */ -static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, - struct kbase_context *kctx) +#else /* KBASE_KTRACE_ENABLE */ +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) { - CSTD_UNUSED(kbdev); CSTD_UNUSED(kctx); return 0; } -#endif /* KBASE_TRACE_ENABLE */ +#endif /* KBASE_KTRACE_ENABLE */ /* * Private functions @@ -187,7 +184,7 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - for (prio = BASE_JD_PRIO_MEDIUM; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) return false; @@ -280,7 +277,7 @@ jsctx_queue_foreach(struct kbase_context *kctx, int js, { int prio; - for (prio = BASE_JD_PRIO_MEDIUM; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) jsctx_queue_foreach_prio(kctx, js, prio, callback); } @@ -336,7 +333,7 @@ jsctx_rb_peek(struct kbase_context *kctx, int js) lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - for (prio = BASE_JD_PRIO_MEDIUM; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { struct kbase_jd_atom *katom; @@ -538,7 +535,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) mutex_init(&jsdd->runpool_mutex); mutex_init(&jsdd->queue_mutex); - spin_lock_init(&kbdev->hwaccess_lock); sema_init(&jsdd->schedule_sem, 1); for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { @@ -1185,7 +1181,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) /* Determine the new priority for context, as per the priority * of currently in-use atoms. */ - for (prio = BASE_JD_PRIO_MEDIUM; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { if (kctx->atoms_count[prio]) { new_priority = prio; @@ -1217,8 +1213,9 @@ static int js_add_start_rp(struct kbase_jd_atom *const start_katom) if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS) return -EINVAL; - if (start_katom->renderpass_id >= ARRAY_SIZE(kctx->jctx.renderpasses)) - return -EINVAL; + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; @@ -1262,8 +1259,9 @@ static int js_add_end_rp(struct kbase_jd_atom *const end_katom) if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS) return -EINVAL; - if (end_katom->renderpass_id >= ARRAY_SIZE(kctx->jctx.renderpasses)) - return -EINVAL; + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; @@ -1376,8 +1374,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx, enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, - kbasep_js_trace_get_refcnt(kbdev, kctx)); + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, + kbase_ktrace_get_ctx_refcnt(kctx)); /* Context Attribute Refcounting */ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); @@ -1443,8 +1441,8 @@ void kbasep_js_remove_job(struct kbase_device *kbdev, js_kctx_info = &kctx->jctx.sched_info; - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, - kbasep_js_trace_get_refcnt(kbdev, kctx)); + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, + kbase_ktrace_get_ctx_refcnt(kctx)); /* De-refcount ctx.nr_jobs */ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); @@ -1530,7 +1528,7 @@ static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( * run more jobs than before */ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, kctx, NULL, 0u, 0); } return result; @@ -1608,10 +1606,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( kbdev, kctx, katom_retained_state); - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, - new_ref_count); - if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && +#ifdef CONFIG_MALI_ARBITER_SUPPORT + !kbase_pm_is_gpu_lost(kbdev) && +#endif !kbase_pm_is_suspending(kbdev)) { /* Context is kept scheduled into an address space even when * there are no jobs, in this case we have to handle the @@ -1629,7 +1627,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( * which was previously acquired by kbasep_js_schedule_ctx(). */ if (new_ref_count == 1 && (!kbasep_js_is_submit_allowed(js_devdata, kctx) || - kbdev->pm.suspending)) { +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_pm_is_gpu_lost(kbdev) || +#endif + kbase_pm_is_suspending(kbdev))) { int num_slots = kbdev->gpu_props.num_job_slots; int slot; @@ -1900,9 +1901,9 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, return false; } - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, 0u, - kbasep_js_trace_get_refcnt(kbdev, kctx)); + kbase_ktrace_get_ctx_refcnt(kctx)); kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); @@ -1939,11 +1940,15 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, * kbasep_js_suspend() code will cleanup this context instead (by virtue * of it being called strictly after the suspend flag is set, and will * wait for this lock to drop) */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { +#else if (kbase_pm_is_suspending(kbdev)) { +#endif /* Cause it to leave at some later point */ bool retained; - retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + retained = kbase_ctx_sched_inc_refcount_nolock(kctx); KBASE_DEBUG_ASSERT(retained); kbasep_js_clear_submit_allowed(js_devdata, kctx); @@ -2016,9 +2021,30 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, js_devdata = &kbdev->js_data; js_kctx_info = &kctx->jctx.sched_info; - /* This must never be attempted whilst suspending - i.e. it should only - * happen in response to a syscall from a user-space thread */ - BUG_ON(kbase_pm_is_suspending(kbdev)); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* This should only happen in response to a system call + * from a user-space thread. + * In a non-arbitrated environment this can never happen + * whilst suspending. + * + * In an arbitrated environment, user-space threads can run + * while we are suspended (for example GPU not available + * to this VM), however in that case we will block on + * the wait event for KCTX_SCHEDULED, since no context + * can be scheduled until we have the GPU again. + */ + if (kbdev->arb.arb_if == NULL) + if (WARN_ON(kbase_pm_is_suspending(kbdev))) + return; +#else + /* This should only happen in response to a system call + * from a user-space thread. + * In a non-arbitrated environment this can never happen + * whilst suspending. + */ + if (WARN_ON(kbase_pm_is_suspending(kbdev))) + return; +#endif mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); @@ -2046,7 +2072,7 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, } else { /* Already scheduled in - We need to retain it to keep the * corresponding address space */ - WARN_ON(!kbasep_js_runpool_retain_ctx(kbdev, kctx)); + WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); } @@ -2136,11 +2162,12 @@ void kbasep_js_resume(struct kbase_device *kbdev) mutex_lock(&js_devdata->queue_mutex); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - for (prio = BASE_JD_PRIO_MEDIUM; + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { struct kbase_context *kctx, *n; unsigned long flags; +#ifndef CONFIG_MALI_ARBITER_SUPPORT spin_lock_irqsave(&kbdev->hwaccess_lock, flags); list_for_each_entry_safe(kctx, n, @@ -2164,9 +2191,13 @@ void kbasep_js_resume(struct kbase_device *kbdev) timer_sync = kbase_js_ctx_list_add_pullable_nolock( kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + if (timer_sync) kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); @@ -2174,6 +2205,30 @@ void kbasep_js_resume(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#else + bool timer_sync = false; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + list_for_each_entry_safe(kctx, n, + &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (timer_sync) { + mutex_lock(&js_devdata->runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + } +#endif } } mutex_unlock(&js_devdata->queue_mutex); @@ -2354,6 +2409,8 @@ static void kbase_js_evict_deps(struct kbase_context *kctx, KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { /* Remove dependency.*/ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + trace_sysgraph(SGR_DEP_RES, kctx->id, + kbase_jd_atom_id(kctx, x_dep)); dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %p\n", (void *)x_dep); @@ -2388,7 +2445,11 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) (void *)kctx); return NULL; } +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) +#else if (kbase_pm_is_suspending(kbdev)) +#endif return NULL; katom = jsctx_rb_peek(kctx, js); @@ -2451,7 +2512,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; jsctx_rb_pull(kctx, katom); - kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + kbase_ctx_sched_retain_ctx_refcount(kctx); katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; @@ -2490,9 +2551,9 @@ static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) return; - if (WARN_ON(start_katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return; + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; @@ -2599,9 +2660,9 @@ static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) return; - if (WARN_ON(end_katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return; + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; @@ -2856,9 +2917,9 @@ static bool js_complete_start_rp(struct kbase_context *kctx, if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) return false; - if (WARN_ON(start_katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return false; + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; @@ -2952,9 +3013,9 @@ static void js_complete_end_rp(struct kbase_context *kctx, if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) return; - if (WARN_ON(end_katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return; + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; @@ -3129,9 +3190,9 @@ static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) return true; - if (WARN_ON(end_katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return true; + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; @@ -3194,6 +3255,9 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, katom->slot_nr, 0, TL_JS_EVENT_STOP); + trace_sysgraph_gpu(SGR_COMPLETE, kctx->id, + kbase_jd_atom_id(katom->kctx, katom), katom->slot_nr); + kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); /* Unblock cross dependency if present */ @@ -3203,7 +3267,8 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, false); x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - + trace_sysgraph(SGR_DEP_RES, kctx->id, + kbase_jd_atom_id(katom->kctx, x_dep)); dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %p\n", (void *)x_dep); @@ -3263,9 +3328,9 @@ bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) return true; } - if (WARN_ON(katom->renderpass_id >= - ARRAY_SIZE(kctx->jctx.renderpasses))) - return true; + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); rp = &kctx->jctx.renderpasses[katom->renderpass_id]; /* We can read a subset of renderpass state without holding @@ -3399,11 +3464,12 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) kbase_ctx_flag_clear(kctx, KCTX_PULLED); - if (!kbase_jm_kick(kbdev, 1 << js)) + if (!kbase_jm_kick(kbdev, 1 << js)) { dev_dbg(kbdev->dev, "No more jobs can be submitted (s:%d)\n", js); js_mask &= ~(1 << js); + } if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { bool pullable; @@ -3582,8 +3648,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) * back (this already cancels the jobs) */ - KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); @@ -3603,8 +3668,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* Case c: didn't evict, but it is scheduled - it's in the Run * Pool */ - KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); /* Disable the ctx from submitting any more jobs */ @@ -3615,7 +3679,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* Retain and (later) release the context whilst it is is now * disallowed from submitting jobs - ensures that someone * somewhere will be removing the context later on */ - was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); /* Since it's scheduled and we have the jsctx_mutex, it must be * retained successfully */ @@ -3639,7 +3703,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) kbasep_js_runpool_release_ctx(kbdev, kctx); } - KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); /* After this, you must wait on both the * kbase_jd_context::zero_jobs_wait and the @@ -3685,7 +3749,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, 0u, trace_get_refcnt(kbdev, kctx)); /* Invoke callback on jobs on each slot in turn */ @@ -3694,81 +3758,3 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } - - -/* Hold the mmu_hw_mutex and hwaccess_lock for this */ -bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - bool result = false; - int as_nr; - - if (WARN_ON(kbdev == NULL)) - return result; - - if (WARN_ON(kctx == NULL)) - return result; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - as_nr = kctx->as_nr; - if (atomic_read(&kctx->refcount) > 0) { - KBASE_DEBUG_ASSERT(as_nr >= 0); - - kbase_ctx_sched_retain_ctx_refcount(kctx); - KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, - NULL, 0u, atomic_read(&kctx->refcount)); - result = true; - } - - return result; -} - - -bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - unsigned long flags; - bool result = false; - - if (WARN_ON(kbdev == NULL)) - return result; - - if (WARN_ON(kctx == NULL)) - return result; - - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - return result; -} - -struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, - int as_nr) -{ - unsigned long flags; - struct kbase_context *found_kctx = NULL; - - if (WARN_ON(kbdev == NULL)) - return NULL; - - if (WARN_ON(as_nr < 0)) - return NULL; - - if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) - return NULL; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - found_kctx = kbdev->as_to_kctx[as_nr]; - - if (found_kctx != NULL) - kbase_ctx_sched_retain_ctx_refcount(found_kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return found_kctx; -} diff --git a/mali_kbase/mali_kbase_js.h b/mali_kbase/mali_kbase_js.h index 51ab023..541acd4 100644 --- a/mali_kbase/mali_kbase_js.h +++ b/mali_kbase/mali_kbase_js.h @@ -30,100 +30,11 @@ #ifndef _KBASE_JS_H_ #define _KBASE_JS_H_ -#include "mali_kbase_js_defs.h" #include "context/mali_kbase_context.h" #include "mali_kbase_defs.h" #include "mali_kbase_debug.h" #include <mali_kbase_ctx_sched.h> - -#include <jm/mali_kbase_jm_js.h> - -/** - * kbasep_js_runpool_retain_ctx_nolock - Refcount a context as being busy, - * preventing it from being scheduled - * out. - * - * This function can safely be called from IRQ context. - * - * The following locks must be held by the caller: - * * mmu_hw_mutex, hwaccess_lock - * - * Return: value true if the retain succeeded, and the context will not be - * scheduled out, otherwise false if the retain failed (because the context - * is being/has been scheduled out). - */ -bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx); - -/** - * kbasep_js_runpool_retain_ctx - Refcount a context as being busy, preventing - * it from being scheduled out. - * - * This function can safely be called from IRQ context. - * - * The following locking conditions are made on the caller: - * * it must not hold mmu_hw_mutex and hwaccess_lock, because they will be - * used internally. - * - * Return: value true if the retain succeeded, and the context will not be - * scheduled out, otherwise false if the retain failed (because the context - * is being/has been scheduled out). - */ -bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, - struct kbase_context *kctx); - -/** - * kbasep_js_runpool_lookup_ctx - Lookup a context in the Run Pool based upon - * its current address space and ensure that - * is stays scheduled in. - * - * The context is refcounted as being busy to prevent it from scheduling - * out. It must be released with kbasep_js_runpool_release_ctx() when it is no - * longer required to stay scheduled in. - * - * This function can safely be called from IRQ context. - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * If the hwaccess_lock is already held, then the caller should use - * kbasep_js_runpool_lookup_ctx_nolock() instead. - * - * Return: a valid struct kbase_context on success, which has been refcounted - * as being busy or return NULL on failure, indicating that no context was found - * in as_nr. - */ -struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, - int as_nr); - -/** - * kbasep_js_runpool_lookup_ctx_noretain - Variant of - * kbasep_js_runpool_lookup_ctx() that can be used when the - * context is guaranteed to be already previously retained. - * - * It is a programming error to supply the as_nr of a context that has not - * been previously retained/has a busy refcount of zero. The only exception is - * when there is no ctx in as_nr (NULL returned). - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * - * Return: a valid struct kbase_context on success, with a refcount that is - * guaranteed to be non-zero and unmodified by this function or - * return NULL on failure, indicating that no context was found in as_nr. - */ -static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain( - struct kbase_device *kbdev, int as_nr) -{ - struct kbase_context *found_kctx; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); - - found_kctx = kbdev->as_to_kctx[as_nr]; - KBASE_DEBUG_ASSERT(found_kctx == NULL || - atomic_read(&found_kctx->refcount) > 0); - - return found_kctx; -} +#include "jm/mali_kbase_jm_js.h" +#include "jm/mali_kbase_js_defs.h" #endif /* _KBASE_JS_H_ */ diff --git a/mali_kbase/mali_kbase_js_ctx_attr.c b/mali_kbase/mali_kbase_js_ctx_attr.c index 1ff230c..141d04a 100644 --- a/mali_kbase/mali_kbase_js_ctx_attr.c +++ b/mali_kbase/mali_kbase_js_ctx_attr.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -67,7 +67,7 @@ static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, s if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { /* First refcount indicates a state change */ runpool_state_changed = true; - KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); } } @@ -112,7 +112,7 @@ static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { /* Last de-refcount indicates a state change */ runpool_state_changed = true; - KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); } } @@ -149,7 +149,7 @@ static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struc if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ - KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); } @@ -185,7 +185,7 @@ static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, stru lockdep_assert_held(&kbdev->hwaccess_lock); /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); - KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); } /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c index 2362e22..4a1004b 100644 --- a/mali_kbase/mali_kbase_mem.c +++ b/mali_kbase/mali_kbase_mem.c @@ -2983,6 +2983,63 @@ static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, kbase_jit_debugfs_phys_get); +#if MALI_JIT_PRESSURE_LIMIT +static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jctx.lock); + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->used_pages; + } + mutex_unlock(&kctx->jit_evict_lock); + mutex_unlock(&kctx->jctx.lock); + + return 0; +} + +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_used_fops, + kbase_jit_debugfs_used_get); + +static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, + struct kbase_va_region *reg, size_t pages_needed, + size_t *freed, bool shrink); + +static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jctx.lock); + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + int err; + size_t freed = 0u; + + err = kbase_mem_jit_trim_pages_from_region(kctx, reg, + SIZE_MAX, &freed, false); + + if (err) { + /* Failed to calculate, try the next region */ + continue; + } + + data->active_value += freed; + } + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); + mutex_unlock(&kctx->jctx.lock); + + return 0; +} + +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_trim_fops, + kbase_jit_debugfs_trim_get); +#endif /* MALI_JIT_PRESSURE_LIMIT */ + void kbase_jit_debugfs_init(struct kbase_context *kctx) { /* prevent unprivileged use of debug file system @@ -3021,6 +3078,22 @@ void kbase_jit_debugfs_init(struct kbase_context *kctx) */ debugfs_create_file("mem_jit_phys", mode, kctx->kctx_dentry, kctx, &kbase_jit_debugfs_phys_fops); +#if MALI_JIT_PRESSURE_LIMIT + /* + * Debugfs entry for getting the number of pages used + * by JIT allocations for estimating the physical pressure + * limit. + */ + debugfs_create_file("mem_jit_used", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_used_fops); + + /* + * Debugfs entry for getting the number of pages that could + * be trimmed to free space for more JIT allocations. + */ + debugfs_create_file("mem_jit_trim", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_trim_fops); +#endif /* MALI_JIT_PRESSURE_LIMIT */ } #endif /* CONFIG_DEBUG_FS */ @@ -3065,8 +3138,8 @@ int kbase_jit_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->jit_destroy_head); INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); - INIT_LIST_HEAD(&kctx->jit_pending_alloc); - INIT_LIST_HEAD(&kctx->jit_atoms_head); + INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); + INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); mutex_unlock(&kctx->jit_evict_lock); kctx->jit_max_allocations = 0; @@ -3081,7 +3154,7 @@ int kbase_jit_init(struct kbase_context *kctx) * the alignment requirements. */ static bool meet_size_and_tiler_align_top_requirements(struct kbase_context *kctx, - struct kbase_va_region *walker, struct base_jit_alloc_info *info) + struct kbase_va_region *walker, const struct base_jit_alloc_info *info) { bool meet_reqs = true; @@ -3103,7 +3176,7 @@ static bool meet_size_and_tiler_align_top_requirements(struct kbase_context *kct */ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, struct kbase_va_region *reg, size_t pages_needed, - size_t *freed) + size_t *freed, bool shrink) { int err = 0; size_t available_pages = 0u; @@ -3199,10 +3272,11 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, available_pages = old_pages - reg->used_pages; to_free = min(available_pages, pages_needed); - new_pages -= to_free; - - err = kbase_mem_shrink(kctx, reg, new_pages); + if (shrink) { + new_pages -= to_free; + err = kbase_mem_shrink(kctx, reg, new_pages); + } out: trace_mali_jit_trim_from_region(reg, to_free, old_pages, available_pages, new_pages); @@ -3210,7 +3284,25 @@ out: return err; } -size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, + +/** + * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been + * freed + * @kctx: Pointer to the kbase context whose active JIT allocations will be + * checked. + * @pages_needed: The maximum number of pages to trim. + * + * This functions checks all active JIT allocations in @kctx for unused pages + * at the end, and trim the backed memory regions of those allocations down to + * the used portion and free the unused pages into the page pool. + * + * Specifying @pages_needed allows us to stop early when there's enough + * physical memory freed to sufficiently bring down the total JIT physical page + * usage (e.g. to below the pressure limit) + * + * Return: Total number of successfully freed pages + */ +static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, size_t pages_needed) { struct kbase_va_region *reg, *tmp; @@ -3223,7 +3315,7 @@ size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, size_t freed = 0u; err = kbase_mem_jit_trim_pages_from_region(kctx, reg, - pages_needed, &freed); + pages_needed, &freed, true); if (err) { /* Failed to trim, try the next region */ @@ -3246,7 +3338,8 @@ size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, #endif /* MALI_JIT_PRESSURE_LIMIT */ static int kbase_jit_grow(struct kbase_context *kctx, - struct base_jit_alloc_info *info, struct kbase_va_region *reg) + const struct base_jit_alloc_info *info, + struct kbase_va_region *reg) { size_t delta; size_t pages_required; @@ -3399,30 +3492,143 @@ static void trace_jit_stats(struct kbase_context *kctx, max_allocations, alloc_count, va_pages, ph_pages); } -struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - struct base_jit_alloc_info *info) +#if MALI_JIT_PRESSURE_LIMIT +/** + * get_jit_backed_pressure() - calculate the physical backing of all JIT + * allocations + * + * @kctx: Pointer to the kbase context whose active JIT allocations will be + * checked + * + * Return: number of pages that are committed by JIT allocations + */ +static size_t get_jit_backed_pressure(struct kbase_context *kctx) { - struct kbase_va_region *reg = NULL; + size_t backed_pressure = 0; + int jit_id; + + lockdep_assert_held(&kctx->jctx.lock); + + kbase_gpu_vm_lock(kctx); + for (jit_id = 0; jit_id <= BASE_JIT_ALLOC_COUNT; jit_id++) { + struct kbase_va_region *reg = kctx->jit_alloc[jit_id]; + + if (reg && (reg != KBASE_RESERVED_REG_JIT_ALLOC)) { + /* If region has no report, be pessimistic */ + if (reg->used_pages == reg->nr_pages) { + backed_pressure += reg->nr_pages; + } else { + backed_pressure += + kbase_reg_current_backed_size(reg); + } + } + } + kbase_gpu_vm_unlock(kctx); + + return backed_pressure; +} + +/** + * jit_trim_necessary_pages() - calculate and trim the least pages possible to + * satisfy a new JIT allocation + * + * @kctx: Pointer to the kbase context + * @info: Pointer to JIT allocation information for the new allocation + * + * Before allocating a new just-in-time memory region or reusing a previous + * one, ensure that the total JIT physical page usage also will not exceed the + * pressure limit. + * + * If there are no reported-on allocations, then we already guarantee this will + * be the case - because our current pressure then only comes from the va_pages + * of each JIT region, hence JIT physical page usage is guaranteed to be + * bounded by this. + * + * However as soon as JIT allocations become "reported on", the pressure is + * lowered to allow new JIT regions to be allocated. It is after such a point + * that the total JIT physical page usage could (either now or in the future on + * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly + * allocated JIT regions. Hence, trim any "reported on" regions. + * + * Any pages freed will go into the pool and be allocated from there in + * kbase_mem_alloc(). + */ +static void jit_trim_necessary_pages(struct kbase_context *kctx, + const struct base_jit_alloc_info *info) +{ + size_t backed_pressure = 0; + size_t needed_pages = 0; + + backed_pressure = get_jit_backed_pressure(kctx); + + /* It is possible that this is the case - if this is the first + * allocation after "ignore_pressure_limit" allocation. + */ + if (backed_pressure > kctx->jit_phys_pages_limit) { + needed_pages += + (backed_pressure - kctx->jit_phys_pages_limit) + + info->va_pages; + } else { + size_t backed_diff = + kctx->jit_phys_pages_limit - backed_pressure; + + if (info->va_pages > backed_diff) + needed_pages += info->va_pages - backed_diff; + } + + if (needed_pages) { + size_t trimmed_pages = kbase_mem_jit_trim_pages(kctx, + needed_pages); + + /* This should never happen - we already asserted that + * we are not violating JIT pressure limit in earlier + * checks, which means that in-flight JIT allocations + * must have enough unused pages to satisfy the new + * allocation + */ + WARN_ON(trimmed_pages < needed_pages); + } +} +#endif /* MALI_JIT_PRESSURE_LIMIT */ + +/** + * jit_allow_allocate() - check whether basic conditions are satisfied to allow + * a new JIT allocation + * + * @kctx: Pointer to the kbase context + * @info: Pointer to JIT allocation information for the new allocation + * @ignore_pressure_limit: Flag to indicate whether JIT pressure limit check + * should be ignored + * + * Return: true if allocation can be executed, false otherwise + */ +static bool jit_allow_allocate(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit) +{ + lockdep_assert_held(&kctx->jctx.lock); #if MALI_JIT_PRESSURE_LIMIT - if (info->va_pages > (kctx->jit_phys_pages_limit - - kctx->jit_current_phys_pressure) && - kctx->jit_current_phys_pressure > 0) { + if (likely(!ignore_pressure_limit) && + ((kctx->jit_phys_pages_limit <= kctx->jit_current_phys_pressure) || + (info->va_pages > (kctx->jit_phys_pages_limit - kctx->jit_current_phys_pressure)))) { dev_dbg(kctx->kbdev->dev, "Max JIT page allocations limit reached: active pages %llu, max pages %llu\n", kctx->jit_current_phys_pressure + info->va_pages, kctx->jit_phys_pages_limit); - return NULL; + return false; } #endif /* MALI_JIT_PRESSURE_LIMIT */ + if (kctx->jit_current_allocations >= kctx->jit_max_allocations) { /* Too many current allocations */ dev_dbg(kctx->kbdev->dev, "Max JIT allocations limit reached: active allocations %d, max allocations %d\n", kctx->jit_current_allocations, kctx->jit_max_allocations); - return NULL; + return false; } + if (info->max_allocations > 0 && kctx->jit_current_allocations_per_bin[info->bin_id] >= info->max_allocations) { @@ -3432,34 +3638,26 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, info->bin_id, kctx->jit_current_allocations_per_bin[info->bin_id], info->max_allocations); - return NULL; + return false; } + return true; +} + +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit) +{ + struct kbase_va_region *reg = NULL; + + lockdep_assert_held(&kctx->jctx.lock); + + if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) + return NULL; + #if MALI_JIT_PRESSURE_LIMIT - /* Before allocating a new just-in-time memory region or reusing a - * previous one, ensure that the total JIT physical page usage also will - * not exceed the pressure limit. - * - * If there are no reported-on allocations, then we already guarantee - * this will be the case - because our current pressure then only comes - * from the va_pages of each JIT region, hence JIT physical page usage - * is guaranteed to be bounded by this. - * - * However as soon as JIT allocations become "reported on", the - * pressure is lowered to allow new JIT regions to be allocated. It is - * after such a point that the total JIT physical page usage could - * (either now or in the future on a grow-on-GPU-page-fault) exceed the - * pressure limit, but only on newly allocated JIT regions. Hence, trim - * any "reported on" regions. - * - * Any pages freed will go into the pool and be allocated from there in - * kbase_mem_alloc(). - * - * In future, GPUCORE-21217: Only do this when physical page usage - * could exceed the pressure limit, and only trim as much as is - * necessary. - */ - kbase_mem_jit_trim_pages(kctx, SIZE_MAX); + if (!ignore_pressure_limit) + jit_trim_necessary_pages(kctx, info); #endif /* MALI_JIT_PRESSURE_LIMIT */ mutex_lock(&kctx->jit_evict_lock); @@ -3572,7 +3770,10 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, dev_dbg(kctx->kbdev->dev, "JIT allocation resize failed: va_pages 0x%llx, commit_pages 0x%llx\n", info->va_pages, info->commit_pages); - goto update_failed_unlocked; + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); + return NULL; } } else { /* No suitable JIT allocation was found so create a new one */ @@ -3598,7 +3799,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, dev_dbg(kctx->kbdev->dev, "Failed to allocate JIT memory: va_pages 0x%llx, commit_pages 0x%llx\n", info->va_pages, info->commit_pages); - goto out_unlocked; + return NULL; } mutex_lock(&kctx->jit_evict_lock); @@ -3624,13 +3825,6 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, #endif /* MALI_JIT_PRESSURE_LIMIT */ return reg; - -update_failed_unlocked: - mutex_lock(&kctx->jit_evict_lock); - list_move(®->jit_node, &kctx->jit_pool_head); - mutex_unlock(&kctx->jit_evict_lock); -out_unlocked: - return NULL; } void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) @@ -3848,35 +4042,6 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx, kctx->jit_current_phys_pressure -= diff; reg->used_pages = new_used_pages; - - /* In the case of pressure reduced on a free, don't attempt to - * trim the region: it will soon be placed on the evict_list - * so that if we really were close to running out of memory then - * the shrinker can reclaim the memory. - */ - if ((flags & KBASE_JIT_REPORT_ON_ALLOC_OR_FREE) == 0u) { - size_t freed; - int err; - - kbase_gpu_vm_lock(kctx); - /* If this was from an allocation that a single - * BASE_JD_REQ_SOFT_JIT_ALLOC atom that is allowed to - * breach the pressure limit, then check whether we can - * bring the total JIT physical page below (or at least - * nearer) the pressure limit. - * - * In future, GPUCORE-21217: Only do this when physical - * page usage currently exceeds the pressure limit, and - * only trim as much as is necessary. - */ - err = kbase_mem_jit_trim_pages_from_region(kctx, reg, - SIZE_MAX, &freed); - kbase_gpu_vm_unlock(kctx); - - CSTD_UNUSED(freed); - /* Nothing to do if trimming failed */ - CSTD_UNUSED(err); - } } else { /* We increased the number of used pages */ diff = new_used_pages - reg->used_pages; diff --git a/mali_kbase/mali_kbase_mem.h b/mali_kbase/mali_kbase_mem.h index 3f74492..6e921ec 100644 --- a/mali_kbase/mali_kbase_mem.h +++ b/mali_kbase/mali_kbase_mem.h @@ -1457,11 +1457,13 @@ int kbase_jit_init(struct kbase_context *kctx); * kbase_jit_allocate - Allocate JIT memory * @kctx: kbase context * @info: JIT allocation information + * @ignore_pressure_limit: Whether the JIT memory pressure limit is ignored * * Return: JIT allocation on success or NULL on failure. */ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - struct base_jit_alloc_info *info); + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit); /** * kbase_jit_free - Free a JIT allocation @@ -1560,26 +1562,6 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, void kbase_jit_report_update_pressure(struct kbase_context *kctx, struct kbase_va_region *reg, u64 new_used_pages, unsigned int flags); - -/** - * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been - * freed - * @kctx: Pointer to the kbase context whose active JIT allocations will be - * checked. - * @pages_needed: The maximum number of pages to trim. - * - * This functions checks all active JIT allocations in @kctx for unused pages - * at the end, and trim the backed memory regions of those allocations down to - * the used portion and free the unused pages into the page pool. - * - * Specifying @pages_needed allows us to stop early when there's enough - * physical memory freed to sufficiently bring down the total JIT physical page - * usage (e.g. to below the pressure limit) - * - * Return: Total number of successfully freed pages - */ -size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, - size_t pages_needed); #endif /* MALI_JIT_PRESSURE_LIMIT */ /** diff --git a/mali_kbase/mali_kbase_pm.c b/mali_kbase/mali_kbase_pm.c index 2adbb21..b9ed8c3 100644 --- a/mali_kbase/mali_kbase_pm.c +++ b/mali_kbase/mali_kbase_pm.c @@ -33,6 +33,11 @@ #include <mali_kbase_hwcnt_context.h> #include <mali_kbase_pm.h> +#include <mali_kbase_pm_internal.h> + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include <arbiter/mali_kbase_arbiter_pm.h> +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) { @@ -46,27 +51,36 @@ void kbase_pm_halt(struct kbase_device *kbdev) void kbase_pm_context_active(struct kbase_device *kbdev) { - (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); + (void)kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); } -int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; int c; KBASE_DEBUG_ASSERT(kbdev != NULL); + dev_dbg(kbdev->dev, "%s - reason = %d, pid = %d\n", __func__, + suspend_handler, current->pid); + kbase_pm_lock(kbdev); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) + return 1; + + if (kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev)) { +#else if (kbase_pm_is_suspending(kbdev)) { +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ switch (suspend_handler) { case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: if (kbdev->pm.active_count != 0) break; /* FALLTHROUGH */ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); return 1; case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: @@ -77,16 +91,20 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbas } } c = ++kbdev->pm.active_count; - KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, c); if (c == 1) { - /* First context active: Power on the GPU and any cores requested by - * the policy */ + /* First context active: Power on the GPU and + * any cores requested by the policy + */ kbase_hwaccess_pm_gpu_active(kbdev); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); + dev_dbg(kbdev->dev, "%s %d\n", __func__, kbdev->pm.active_count); return 0; } @@ -95,17 +113,15 @@ KBASE_EXPORT_TEST_API(kbase_pm_context_active); void kbase_pm_context_idle(struct kbase_device *kbdev) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; int c; KBASE_DEBUG_ASSERT(kbdev != NULL); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); + kbase_pm_lock(kbdev); c = --kbdev->pm.active_count; - KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, c); KBASE_DEBUG_ASSERT(c >= 0); @@ -113,20 +129,21 @@ void kbase_pm_context_idle(struct kbase_device *kbdev) /* Last context has gone idle */ kbase_hwaccess_pm_gpu_idle(kbdev); - /* Wake up anyone waiting for this to become 0 (e.g. suspend). The - * waiters must synchronize with us by locking the pm.lock after - * waiting. + /* Wake up anyone waiting for this to become 0 (e.g. suspend). + * The waiters must synchronize with us by locking the pm.lock + * after waiting. */ wake_up(&kbdev->pm.zero_active_count_wait); } - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); + kbase_pm_unlock(kbdev); + dev_dbg(kbdev->dev, "%s %d (pid = %d)\n", __func__, + kbdev->pm.active_count, current->pid); } KBASE_EXPORT_TEST_API(kbase_pm_context_idle); -void kbase_pm_suspend(struct kbase_device *kbdev) +void kbase_pm_driver_suspend(struct kbase_device *kbdev) { KBASE_DEBUG_ASSERT(kbdev); @@ -141,13 +158,31 @@ void kbase_pm_suspend(struct kbase_device *kbdev) kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); mutex_lock(&kbdev->pm.lock); - KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + if (WARN_ON(kbase_pm_is_suspending(kbdev))) { + mutex_unlock(&kbdev->pm.lock); + return; + } kbdev->pm.suspending = true; mutex_unlock(&kbdev->pm.lock); - /* From now on, the active count will drop towards zero. Sometimes, it'll - * go up briefly before going down again. However, once it reaches zero it - * will stay there - guaranteeing that we've idled all pm references */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) { + int i; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_disjoint_state_up(kbdev); + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_softstop(kbdev, i, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + /* From now on, the active count will drop towards zero. Sometimes, + * it'll go up briefly before going down again. However, once + * it reaches zero it will stay there - guaranteeing that we've idled + * all pm references + */ /* Suspend job scheduler and associated components, so that it releases all * the PM active count references */ @@ -155,16 +190,29 @@ void kbase_pm_suspend(struct kbase_device *kbdev) /* Wait for the active count to reach zero. This is not the same as * waiting for a power down, since not all policies power down when this - * reaches zero. */ - wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); + * reaches zero. + */ + dev_dbg(kbdev->dev, ">wait_event - waiting for active_count == 0 (pid = %d)\n", + current->pid); + wait_event(kbdev->pm.zero_active_count_wait, + kbdev->pm.active_count == 0); + dev_dbg(kbdev->dev, ">wait_event - waiting done\n"); /* NOTE: We synchronize with anything that was just finishing a - * kbase_pm_context_idle() call by locking the pm.lock below */ - + * kbase_pm_context_idle() call by locking the pm.lock below + */ kbase_hwaccess_pm_suspend(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) { + mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_stopped(kbdev); + mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } -void kbase_pm_resume(struct kbase_device *kbdev) +void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) { unsigned long flags; @@ -172,18 +220,28 @@ void kbase_pm_resume(struct kbase_device *kbdev) kbase_hwaccess_pm_resume(kbdev); /* Initial active call, to power on the GPU/cores if needed */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + (void)kbase_pm_context_active_handle_suspend(kbdev, + (arb_gpu_start ? + KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE)); +#else kbase_pm_context_active(kbdev); +#endif /* Resume any blocked atoms (which may cause contexts to be scheduled in - * and dependent atoms to run) */ + * and dependent atoms to run) + */ kbase_resume_suspended_soft_jobs(kbdev); /* Resume the Job Scheduler and associated components, and start running - * atoms */ + * atoms + */ kbasep_js_resume(kbdev); /* Matching idle call, to power off the GPU/cores if we didn't actually - * need it and the policy doesn't want it on */ + * need it and the policy doesn't want it on + */ kbase_pm_context_idle(kbdev); /* Re-enable GPU hardware counters */ @@ -194,3 +252,27 @@ void kbase_pm_resume(struct kbase_device *kbdev) /* Resume vinstr */ kbase_vinstr_resume(kbdev->vinstr_ctx); } + +void kbase_pm_suspend(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); + else + kbase_pm_driver_suspend(kbdev); +#else + kbase_pm_driver_suspend(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +} + +void kbase_pm_resume(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT); + else + kbase_pm_driver_resume(kbdev, false); +#else + kbase_pm_driver_resume(kbdev, false); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +} diff --git a/mali_kbase/mali_kbase_pm.h b/mali_kbase/mali_kbase_pm.h index 59a0314..257f959 100644 --- a/mali_kbase/mali_kbase_pm.h +++ b/mali_kbase/mali_kbase_pm.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2015,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,7 +40,8 @@ * * Must be called before any other power management function * - * @param kbdev The kbase device structure for the device (must be a valid pointer) + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) * * @return 0 if the power management framework was successfully initialized. */ @@ -78,8 +79,9 @@ void kbase_pm_term(struct kbase_device *kbdev); /** Increment the count of active contexts. * - * This function should be called when a context is about to submit a job. It informs the active power policy that the - * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. + * This function should be called when a context is about to submit a job. + * It informs the active power policy that the GPU is going to be in use shortly + * and the policy is expected to start turning on the GPU. * * This function will block until the GPU is available. * @@ -98,16 +100,24 @@ void kbase_pm_context_active(struct kbase_device *kbdev); /** Handler codes for doing kbase_pm_context_active_handle_suspend() */ enum kbase_pm_suspend_handler { /** A suspend is not expected/not possible - this is the same as - * kbase_pm_context_active() */ + * kbase_pm_context_active() + */ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, /** If we're suspending, fail and don't increase the active count */ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, - /** If we're suspending, succeed and allow the active count to increase iff - * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). + /** If we're suspending, succeed and allow the active count to increase + * if it didn't go from 0->1 (i.e., we didn't re-activate the GPU). * * This should only be used when there is a bounded time on the activation - * (e.g. guarantee it's going to be idled very soon after) */ - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE + * (e.g. guarantee it's going to be idled very soon after) + */ + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /** Special case when Arbiter has notified we can use GPU. + * Active count should always start at 0 in this case. + */ + KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED, +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ }; /** Suspend 'safe' variant of kbase_pm_context_active() @@ -129,8 +139,9 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbas /** Decrement the reference count of active contexts. * - * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power - * policy so the calling code should ensure that it does not access the GPU's registers. + * This function should be called when a context becomes idle. + * After this call the GPU may be turned off by the power policy so the calling + * code should ensure that it does not access the GPU's registers. * * @param kbdev The kbase device structure for the device (must be a valid pointer) */ @@ -162,6 +173,8 @@ void kbase_pm_suspend(struct kbase_device *kbdev); * This is called in response to an OS resume event, and calls into the various * kbase components to complete the resume. * + * Also called when using VM arbiter, when GPU access has been granted. + * * @param kbdev The kbase device structure for the device (must be a valid pointer) */ void kbase_pm_resume(struct kbase_device *kbdev); @@ -177,4 +190,55 @@ void kbase_pm_resume(struct kbase_device *kbdev); */ void kbase_pm_vsync_callback(int buffer_updated, void *data); -#endif /* _KBASE_PM_H_ */ +/** + * kbase_pm_driver_suspend() - Put GPU and driver in suspend state + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) + * + * Suspend the GPU and prevent any further register accesses to it from Kernel + * threads. + * + * This is called in response to an OS suspend event, and calls into the various + * kbase components to complete the suspend. + * + * Despite kbase_pm_suspend(), it will ignore to update Arbiter + * status if MALI_ARBITER_SUPPORT is enabled. + * + * @note the mechanisms used here rely on all user-space threads being frozen + * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up + * the GPU e.g. via atom submission. + */ +void kbase_pm_driver_suspend(struct kbase_device *kbdev); + +/** + * kbase_pm_driver_resume() - Put GPU and driver in resume + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) + * + * Resume the GPU, allow register accesses to it, and resume running atoms on + * the GPU. + * + * This is called in response to an OS resume event, and calls into the various + * kbase components to complete the resume. + * + * Also called when using VM arbiter, when GPU access has been granted. + * + * Despite kbase_pm_resume(), it will ignore to update Arbiter + * status if MALI_ARBITER_SUPPORT is enabled. + */ +void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +/** + * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM + * @kbdev: Device pointer + * + * Handles the case that the Arbiter has forced the GPU away from the VM, + * so that interrupts will not be received and registers are no longer + * accessible because replaced by dummy RAM. + * Kill any running tasks and put the driver into a GPU powered-off state. + */ +void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +#endif /* _KBASE_PM_H_ */ diff --git a/mali_kbase/mali_kbase_softjobs.c b/mali_kbase/mali_kbase_softjobs.c index 45ce8ad..cbb0c76 100644 --- a/mali_kbase/mali_kbase_softjobs.c +++ b/mali_kbase/mali_kbase_softjobs.c @@ -34,6 +34,7 @@ #include <mali_kbase_hwaccess_time.h> #include <mali_kbase_mem_linux.h> #include <tl/mali_kbase_tracepoints.h> +#include <mali_linux_trace.h> #include <linux/version.h> #include <linux/ktime.h> #include <linux/pfn.h> @@ -133,7 +134,7 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) { struct kbase_vmap_struct map; void *user_result; - struct timespec ts; + struct timespec64 ts; struct base_dump_cpu_gpu_counters data; u64 system_time; u64 cycle_counter; @@ -916,6 +917,12 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, return 0; } + +#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE) +#define offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) +#endif + /* * Sizes of user data to copy for each just-in-time memory interface version * @@ -1000,7 +1007,7 @@ static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) katom->jit_blocked = false; lockdep_assert_held(&kctx->jctx.lock); - list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); /* * Note: @@ -1037,7 +1044,7 @@ static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) struct list_head *target_list_head = NULL; struct kbase_jd_atom *entry; - list_for_each_entry(entry, &kctx->jit_pending_alloc, queue) { + list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) { if (katom->age < entry->age) { target_list_head = &entry->queue; break; @@ -1045,7 +1052,7 @@ static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) } if (target_list_head == NULL) - target_list_head = &kctx->jit_pending_alloc; + target_list_head = &kctx->jctx.jit_pending_alloc; list_add_tail(&katom->queue, target_list_head); } @@ -1060,6 +1067,10 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) u64 *ptr, new_addr; u32 count = katom->nr_extres; u32 i; + bool ignore_pressure_limit = false; + + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom)); if (katom->jit_blocked) { list_del(&katom->queue); @@ -1080,6 +1091,15 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) } } +#if MALI_JIT_PRESSURE_LIMIT + /** + * If this is the only JIT_ALLOC atom in-flight then allow it to exceed + * the defined pressure limit. + */ + if (kctx->jit_current_allocations == 0) + ignore_pressure_limit = true; +#endif /* MALI_JIT_PRESSURE_LIMIT */ + for (i = 0, info = katom->softjob_data; i < count; i++, info++) { if (kctx->jit_alloc[info->id]) { /* The JIT ID is duplicated in this atom. Roll back @@ -1099,17 +1119,14 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) } /* Create a JIT allocation */ - reg = kbase_jit_allocate(kctx, info); + reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit); if (!reg) { struct kbase_jd_atom *jit_atom; bool can_block = false; lockdep_assert_held(&kctx->jctx.lock); - jit_atom = list_first_entry(&kctx->jit_atoms_head, - struct kbase_jd_atom, jit_node); - - list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { + list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) { if (jit_atom == katom) break; @@ -1280,7 +1297,7 @@ static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) for (i = 0; i < count; i++) KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]); - list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); return 0; @@ -1335,7 +1352,7 @@ void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) LIST_HEAD(jit_pending_alloc_list); struct list_head *i, *tmp; - list_splice_tail_init(&kctx->jit_pending_alloc, + list_splice_tail_init(&kctx->jctx.jit_pending_alloc, &jit_pending_alloc_list); list_for_each_safe(i, tmp, &jit_pending_alloc_list) { @@ -1363,7 +1380,7 @@ static void kbase_jit_free_finish(struct kbase_jd_atom *katom) return; } - /* Remove this atom from the kctx->jit_atoms_head list */ + /* Remove this atom from the jit_atoms_head list */ list_del(&katom->jit_node); for (j = 0; j != katom->nr_extres; ++j) { @@ -1522,6 +1539,9 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom) KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom)); + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: ret = kbase_dump_cpu_gpu_time(katom); @@ -1684,6 +1704,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) void kbase_finish_soft_job(struct kbase_jd_atom *katom) { + trace_sysgraph(SGR_COMPLETE, katom->kctx->id, + kbase_jd_atom_id(katom->kctx, katom)); + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: /* Nothing to do */ diff --git a/mali_kbase/mali_kbase_vinstr.c b/mali_kbase/mali_kbase_vinstr.c index f01291a..d96b565 100644 --- a/mali_kbase/mali_kbase_vinstr.c +++ b/mali_kbase/mali_kbase_vinstr.c @@ -139,10 +139,7 @@ static const struct file_operations vinstr_client_fops = { */ static u64 kbasep_vinstr_timestamp_ns(void) { - struct timespec ts; - - getrawmonotonic(&ts); - return (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + return ktime_get_raw_ns(); } /** diff --git a/mali_kbase/mali_linux_kbase_trace.h b/mali_kbase/mali_linux_kbase_trace.h deleted file mode 100644 index 6c6a8c6..0000000 --- a/mali_kbase/mali_linux_kbase_trace.h +++ /dev/null @@ -1,204 +0,0 @@ -/* - * - * (C) COPYRIGHT 2014,2018 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0 - * - */ - - -#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_MALI_KBASE_H - -#undef TRACE_SYSTEM -#define TRACE_SYSTEM mali - -#include <linux/tracepoint.h> - -DECLARE_EVENT_CLASS(mali_slot_template, - TP_PROTO(int jobslot, unsigned int info_val), - TP_ARGS(jobslot, info_val), - TP_STRUCT__entry( - __field(unsigned int, jobslot) - __field(unsigned int, info_val) - ), - TP_fast_assign( - __entry->jobslot = jobslot; - __entry->info_val = info_val; - ), - TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) -); - -#define DEFINE_MALI_SLOT_EVENT(name) \ -DEFINE_EVENT(mali_slot_template, mali_##name, \ - TP_PROTO(int jobslot, unsigned int info_val), \ - TP_ARGS(jobslot, info_val)) -DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); -DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); -DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); -DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); -DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); -DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); -DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); -DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); -DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); -DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); -DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); -DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); -DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); -DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); -DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); -DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); -DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); -DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); -DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); -DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); -DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); -DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); -DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); -DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); -#undef DEFINE_MALI_SLOT_EVENT - -DECLARE_EVENT_CLASS(mali_refcount_template, - TP_PROTO(int refcount, unsigned int info_val), - TP_ARGS(refcount, info_val), - TP_STRUCT__entry( - __field(unsigned int, refcount) - __field(unsigned int, info_val) - ), - TP_fast_assign( - __entry->refcount = refcount; - __entry->info_val = info_val; - ), - TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) -); - -#define DEFINE_MALI_REFCOUNT_EVENT(name) \ -DEFINE_EVENT(mali_refcount_template, mali_##name, \ - TP_PROTO(int refcount, unsigned int info_val), \ - TP_ARGS(refcount, info_val)) -DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); -DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); -DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); -DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); -DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); -DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); -DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); -#undef DEFINE_MALI_REFCOUNT_EVENT - -DECLARE_EVENT_CLASS(mali_add_template, - TP_PROTO(int gpu_addr, unsigned int info_val), - TP_ARGS(gpu_addr, info_val), - TP_STRUCT__entry( - __field(unsigned int, gpu_addr) - __field(unsigned int, info_val) - ), - TP_fast_assign( - __entry->gpu_addr = gpu_addr; - __entry->info_val = info_val; - ), - TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) -); - -#define DEFINE_MALI_ADD_EVENT(name) \ -DEFINE_EVENT(mali_add_template, mali_##name, \ - TP_PROTO(int gpu_addr, unsigned int info_val), \ - TP_ARGS(gpu_addr, info_val)) -DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); -DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); -DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); -DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); -DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); -DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); -DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); -DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); -DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); -DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); -DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); -DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); -DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); -DEFINE_MALI_ADD_EVENT(JD_DONE); -DEFINE_MALI_ADD_EVENT(JD_CANCEL); -DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); -DEFINE_MALI_ADD_EVENT(JM_IRQ); -DEFINE_MALI_ADD_EVENT(JM_IRQ_END); -DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); -DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); -DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); -DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); -DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); -DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); -DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); -DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); -DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); -DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); -DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); -DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); -DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); -DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); -DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); -DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); -DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); -DEFINE_MALI_ADD_EVENT(PM_PWRON); -DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); -DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); -DEFINE_MALI_ADD_EVENT(PM_PWROFF); -DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); -DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); -DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); -DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); -DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); -DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); -DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); -DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); -DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED); -DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED); -DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED); -DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); -DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); -DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); -DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); -DEFINE_MALI_ADD_EVENT(PM_GPU_ON); -DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); -DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); -DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); -DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); -DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); -DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); -#undef DEFINE_MALI_ADD_EVENT - -#endif /* _TRACE_MALI_KBASE_H */ - -#undef TRACE_INCLUDE_PATH -#undef linux -#define TRACE_INCLUDE_PATH . -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE mali_linux_kbase_trace - -/* This part must be outside protection */ -#include <trace/define_trace.h> diff --git a/mali_kbase/mali_linux_trace.h b/mali_kbase/mali_linux_trace.h index b639764..f618755 100644 --- a/mali_kbase/mali_linux_trace.h +++ b/mali_kbase/mali_linux_trace.h @@ -288,7 +288,7 @@ DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); -#if MALI_JIT_PRESSURE_LIMIT && !MALI_USE_CSF +#if MALI_JIT_PRESSURE_LIMIT /* trace_mali_jit_report * * Tracepoint about the GPU data structure read to form a just-in-time memory @@ -326,9 +326,11 @@ TRACE_EVENT(mali_jit_report, ), __entry->read_val, __entry->used_pages) ); -#endif /* MALI_JIT_PRESSURE_LIMIT && !MALI_USE_CSF */ +#endif /* MALI_JIT_PRESSURE_LIMIT */ +#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); +#endif #if MALI_JIT_PRESSURE_LIMIT /* trace_mali_jit_report_pressure @@ -366,6 +368,70 @@ TRACE_EVENT(mali_jit_report_pressure, ); #endif /* MALI_JIT_PRESSURE_LIMIT */ +#ifndef __TRACE_SYSGRAPH_ENUM +#define __TRACE_SYSGRAPH_ENUM +/* Enum of sysgraph message IDs */ +enum sysgraph_msg { + SGR_ARRIVE, + SGR_DEP_RES, + SGR_SUBMIT, + SGR_COMPLETE, + SGR_POST, + SGR_ACTIVE, + SGR_INACTIVE +}; +#endif /* __TRACE_SYSGRAPH_ENUM */ + +/* A template for SYSGRAPH events + * + * Most of the sysgraph events contain only one input argument + * which is atom_id therefore they will be using a common template + */ +TRACE_EVENT(sysgraph, + TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, + unsigned int atom_id), + TP_ARGS(message, proc_id, atom_id), + TP_STRUCT__entry( + __field(unsigned int, proc_id) + __field(enum sysgraph_msg, message) + __field(unsigned int, atom_id) + ), + TP_fast_assign( + __entry->proc_id = proc_id; + __entry->message = message; + __entry->atom_id = atom_id; + ), + TP_printk("msg=%u proc_id=%u, param1=%d\n", __entry->message, + __entry->proc_id, __entry->atom_id) +); + +/* A template for SYSGRAPH GPU events + * + * Sysgraph events that record start/complete events + * on GPU also record a js value in addition to the + * atom id. + */ +TRACE_EVENT(sysgraph_gpu, + TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, + unsigned int atom_id, unsigned int js), + TP_ARGS(message, proc_id, atom_id, js), + TP_STRUCT__entry( + __field(unsigned int, proc_id) + __field(enum sysgraph_msg, message) + __field(unsigned int, atom_id) + __field(unsigned int, js) + ), + TP_fast_assign( + __entry->proc_id = proc_id; + __entry->message = message; + __entry->atom_id = atom_id; + __entry->js = js; + ), + TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d\n", + __entry->message, __entry->proc_id, + __entry->atom_id, __entry->js) +); + /* Tracepoint files get included more than once - protect against multiple * definition */ @@ -450,9 +516,14 @@ TRACE_EVENT(mali_jit_trim, TP_printk("freed_pages=%zu", __entry->freed_pages) ); +#include "mali_kbase_debug_linux_ktrace.h" + #endif /* _TRACE_MALI_H */ #undef TRACE_INCLUDE_PATH +/* lwn.net/Articles/383362 suggests this should remain as '.', and instead + * extend CFLAGS + */ #define TRACE_INCLUDE_PATH . #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE mali_linux_trace diff --git a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c index 46800fe..2d8fb51 100644 --- a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c +++ b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c @@ -30,6 +30,7 @@ #include <backend/gpu/mali_kbase_device_internal.h> #include <mali_kbase_as_fault_debugfs.h> #include "../mali_kbase_mmu_internal.h" +#include "mali_kbase_device_internal.h" void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup * const setup) @@ -295,6 +296,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) unsigned long flags; u32 new_mask; u32 tmp, bf_bits, pf_bits; + bool gpu_lost = false; dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", __func__, irq_stat); @@ -338,7 +340,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) * and a job causing the Bus/Page fault shouldn't complete until * the MMU is updated */ - kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); + kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); /* find faulting address */ fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, @@ -369,6 +371,14 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); } + /* check if we still have GPU */ + gpu_lost = kbase_is_gpu_lost(kbdev); + if (gpu_lost) { + if (kctx) + kbasep_js_runpool_release_ctx(kbdev, kctx); + return; + } + if (kbase_as_has_bus_fault(as, fault)) { /* Mark bus fault as handled. * Note that a bus fault is processed first in case diff --git a/mali_kbase/mmu/mali_kbase_mmu.c b/mali_kbase/mmu/mali_kbase_mmu.c index c4bea39..db27832 100644 --- a/mali_kbase/mmu/mali_kbase_mmu.c +++ b/mali_kbase/mmu/mali_kbase_mmu.c @@ -32,6 +32,7 @@ #include <gpu/mali_kbase_gpu_regmap.h> #include <tl/mali_kbase_tracepoints.h> #include <mali_kbase_instr_defs.h> +#include <mali_kbase_ctx_sched.h> #include <mali_kbase_debug.h> #include <mali_kbase_defs.h> #include <mali_kbase_hw.h> @@ -517,6 +518,13 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, return true; } +/* Small wrapper function to factor out GPU-dependent context releasing */ +static void release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + kbasep_js_runpool_release_ctx(kbdev, kctx); +} + void page_fault_worker(struct work_struct *data) { u64 fault_pfn; @@ -552,8 +560,8 @@ void page_fault_worker(struct work_struct *data) * Therefore, it cannot be scheduled out of this AS until we explicitly * release it */ - kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); - if (WARN_ON(!kctx)) { + kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); + if (!kctx) { atomic_dec(&kbdev->faults_pending); return; } @@ -917,7 +925,7 @@ fault_done: * By this point, the fault was handled in some way, * so release the ctx refcount */ - kbasep_js_runpool_release_ctx(kbdev, kctx); + release_ctx(kbdev, kctx); atomic_dec(&kbdev->faults_pending); dev_dbg(kbdev->dev, "Leaving page_fault_worker %p\n", (void *)data); @@ -1562,7 +1570,7 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, kbdev = kctx->kbdev; mutex_lock(&kbdev->js_data.queue_mutex); - ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); + ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); mutex_unlock(&kbdev->js_data.queue_mutex); if (ctx_is_in_runpool) { @@ -1571,7 +1579,7 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[kctx->as_nr], vpfn, nr, sync); - kbasep_js_runpool_release_ctx(kbdev, kctx); + release_ctx(kbdev, kctx); } } @@ -2146,8 +2154,8 @@ void bus_fault_worker(struct work_struct *data) * flagging of the bus-fault. Therefore, it cannot be scheduled out of * this AS until we explicitly release it */ - kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); - if (WARN_ON(!kctx)) { + kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); + if (!kctx) { atomic_dec(&kbdev->faults_pending); return; } @@ -2157,7 +2165,7 @@ void bus_fault_worker(struct work_struct *data) "Permission failure", fault); kbase_mmu_hw_clear_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - kbasep_js_runpool_release_ctx(kbdev, kctx); + release_ctx(kbdev, kctx); atomic_dec(&kbdev->faults_pending); return; @@ -2172,7 +2180,7 @@ void bus_fault_worker(struct work_struct *data) kbase_pm_context_idle(kbdev); } - kbasep_js_runpool_release_ctx(kbdev, kctx); + release_ctx(kbdev, kctx); atomic_dec(&kbdev->faults_pending); } diff --git a/mali_kbase/tests/Mconfig b/mali_kbase/tests/Mconfig index af4e383..be3fedb 100644 --- a/mali_kbase/tests/Mconfig +++ b/mali_kbase/tests/Mconfig @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2018 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -30,3 +30,9 @@ config BUILD_CSF_TESTS bool default y if UNIT_TEST_KERNEL_MODULES && GPU_HAS_CSF default n + +config BUILD_ARBIF_TESTS + bool + default y if UNIT_TEST_KERNEL_MODULES && MALI_ARBITER_SUPPORT + default n + diff --git a/mali_kbase/tests/mali_kutf_irq_test/Makefile b/mali_kbase/tests/mali_kutf_irq_test/Makefile index 9218a40..bc4d654 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/Makefile +++ b/mali_kbase/tests/mali_kutf_irq_test/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2015, 2017-2018 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2015, 2017-2018, 2020 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -38,6 +38,8 @@ TEST_CCFLAGS := \ -I$(CURDIR)/../../../ \ -I$(CURDIR)/../../ \ -I$(CURDIR)/../../backend/gpu \ + -I$(CURDIR)/../../debug \ + -I$(CURDIR)/../../debug/backend \ -I$(CURDIR)/ \ -I$(srctree)/drivers/staging/android \ -I$(srctree)/include/linux diff --git a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c index a220a46..26b442a 100644 --- a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +++ b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -58,10 +58,10 @@ struct kutf_irq_fixture_data { #define NR_TEST_IRQS ((u32)1000000) -/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not - * expect to see this in normal use (e.g., when Android is running). +/* IRQ for the test to trigger. Currently POWER_CHANGED_SINGLE as it is + * otherwise unused in the DDK */ -#define TEST_IRQ MULTIPLE_GPU_FAULTS +#define TEST_IRQ POWER_CHANGED_SINGLE #define IRQ_TIMEOUT HZ @@ -92,11 +92,11 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) struct kbase_device *kbdev = kbase_untag(data); u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS)); irqreturn_t result; - struct timespec tval; + u64 tval; bool has_test_irq = val & TEST_IRQ; if (has_test_irq) { - getnstimeofday(&tval); + tval = ktime_get_real_ns(); /* Clear the test source only here */ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), TEST_IRQ); @@ -107,7 +107,7 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) result = kbase_gpu_irq_test_handler(irq, data, val); if (has_test_irq) { - irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + irq_time = tval; triggered = true; wake_up(&wait); result = IRQ_HANDLED; @@ -191,12 +191,9 @@ static void mali_kutf_irq_latency(struct kutf_context *context) GPU_IRQ_HANDLER); for (i = 1; i <= NR_TEST_IRQS; i++) { - struct timespec tval; - u64 start_time; + u64 start_time = ktime_get_real_ns(); triggered = false; - getnstimeofday(&tval); - start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); /* Trigger fake IRQ */ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), diff --git a/mali_kbase/tl/mali_kbase_timeline.c b/mali_kbase/tl/mali_kbase_timeline.c index 5d073be..88fba83 100644 --- a/mali_kbase/tl/mali_kbase_timeline.c +++ b/mali_kbase/tl/mali_kbase_timeline.c @@ -105,12 +105,12 @@ static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer) /*****************************************************************************/ int kbase_timeline_init(struct kbase_timeline **timeline, - atomic_t *timeline_is_enabled) + atomic_t *timeline_flags) { enum tl_stream_type i; struct kbase_timeline *result; - if (!timeline || !timeline_is_enabled) + if (!timeline || !timeline_flags) return -EINVAL; result = kzalloc(sizeof(*result), GFP_KERNEL); @@ -129,7 +129,7 @@ int kbase_timeline_init(struct kbase_timeline **timeline, atomic_set(&result->autoflush_timer_active, 0); kbase_timer_setup(&result->autoflush_timer, kbasep_timeline_autoflush_timer_callback); - result->is_enabled = timeline_is_enabled; + result->timeline_flags = timeline_flags; *timeline = result; @@ -176,10 +176,10 @@ static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) { int ret; - u32 tlstream_enabled = TLSTREAM_ENABLED | flags; + u32 timeline_flags = TLSTREAM_ENABLED | flags; struct kbase_timeline *timeline = kbdev->timeline; - if (!atomic_cmpxchg(timeline->is_enabled, 0, tlstream_enabled)) { + if (!atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) { int rcode; ret = anon_inode_getfd( @@ -188,7 +188,7 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) timeline, O_RDONLY | O_CLOEXEC); if (ret < 0) { - atomic_set(timeline->is_enabled, 0); + atomic_set(timeline->timeline_flags, 0); return ret; } diff --git a/mali_kbase/tl/mali_kbase_timeline.h b/mali_kbase/tl/mali_kbase_timeline.h index d800288..cd48411 100644 --- a/mali_kbase/tl/mali_kbase_timeline.h +++ b/mali_kbase/tl/mali_kbase_timeline.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -31,15 +31,15 @@ struct kbase_timeline; /** * kbase_timeline_init - initialize timeline infrastructure in kernel - * @timeline: Newly created instance of kbase_timeline will - * be stored in this pointer. - * @timeline_is_enabled: Timeline status will be written to this variable - * when a client is attached/detached. The variable - * must be valid while timeline instance is valid. + * @timeline: Newly created instance of kbase_timeline will be stored in + * this pointer. + * @timeline_flags: Timeline status will be written to this variable when a + * client is attached/detached. The variable must be valid + * while timeline instance is valid. * Return: zero on success, negative number on error */ int kbase_timeline_init(struct kbase_timeline **timeline, - atomic_t *timeline_is_enabled); + atomic_t *timeline_flags); /** * kbase_timeline_term - terminate timeline infrastructure in kernel diff --git a/mali_kbase/tl/mali_kbase_timeline_io.c b/mali_kbase/tl/mali_kbase_timeline_io.c index 6e09a17..cdde928 100644 --- a/mali_kbase/tl/mali_kbase_timeline_io.c +++ b/mali_kbase/tl/mali_kbase_timeline_io.c @@ -324,6 +324,6 @@ static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) atomic_set(&timeline->autoflush_timer_active, 0); del_timer_sync(&timeline->autoflush_timer); - atomic_set(timeline->is_enabled, 0); + atomic_set(timeline->timeline_flags, 0); return 0; } diff --git a/mali_kbase/tl/mali_kbase_timeline_priv.h b/mali_kbase/tl/mali_kbase_timeline_priv.h index 73499ce..3596584 100644 --- a/mali_kbase/tl/mali_kbase_timeline_priv.h +++ b/mali_kbase/tl/mali_kbase_timeline_priv.h @@ -40,7 +40,7 @@ * have access to the timeline streams at any given time. * @event_queue: Timeline stream event queue * @bytes_collected: Number of bytes read by user - * @is_enabled: Zero, if timeline is disabled. Timeline stream flags + * @timeline_flags: Zero, if timeline is disabled. Timeline stream flags * otherwise. See kbase_timeline_io_acquire(). * @obj_header_btc: Remaining bytes to copy for the object stream header * @aux_header_btc: Remaining bytes to copy for the aux stream header @@ -54,7 +54,7 @@ struct kbase_timeline { #if MALI_UNIT_TEST atomic_t bytes_collected; #endif /* MALI_UNIT_TEST */ - atomic_t *is_enabled; + atomic_t *timeline_flags; size_t obj_header_btc; size_t aux_header_btc; }; diff --git a/mali_kbase/tl/mali_kbase_tl_serialize.h b/mali_kbase/tl/mali_kbase_tl_serialize.h index 90808ce..3e37827 100644 --- a/mali_kbase/tl/mali_kbase_tl_serialize.h +++ b/mali_kbase/tl/mali_kbase_tl_serialize.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -113,11 +113,9 @@ static inline size_t kbasep_serialize_string( */ static inline size_t kbasep_serialize_timestamp(void *buffer, size_t pos) { - struct timespec ts; u64 timestamp; - getrawmonotonic(&ts); - timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; + timestamp = ktime_get_raw_ns(); return kbasep_serialize_bytes( buffer, pos, diff --git a/mali_kbase/tl/mali_kbase_trace_defs.h b/mali_kbase/tl/mali_kbase_trace_defs.h deleted file mode 100644 index 1ee6a59..0000000 --- a/mali_kbase/tl/mali_kbase_trace_defs.h +++ /dev/null @@ -1,261 +0,0 @@ -/* - * - * (C) COPYRIGHT 2011-2015,2018-2019 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0 - * - */ - - - -/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** - * ***** DO NOT INCLUDE DIRECTLY ***** - * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ - -/* - * The purpose of this header file is just to contain a list of trace code idenitifers - * - * Each identifier is wrapped in a macro, so that its string form and enum form can be created - * - * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. - * - * This allows automatic creation of an enum and a corresponding array of strings - * - * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. - * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. - * - * e.g.: - * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X - * typedef enum - * { - * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) - * #include "mali_kbase_trace_defs.h" - * #undef KBASE_TRACE_CODE_MAKE_CODE - * } kbase_trace_code; - * - * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE - * - * - * The use of the macro here is: - * - KBASE_TRACE_CODE_MAKE_CODE( X ) - * - * Which produces: - * - For an enum, KBASE_TRACE_CODE_X - * - For a string, "X" - * - * - * For example: - * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: - * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum - * - "JM_JOB_COMPLETE" for the string - * - To use it to trace an event, do: - * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); - */ - -#if 0 /* Dummy section to avoid breaking formatting */ -int dummy_array[] = { -#endif - -/* - * Core events - */ - /* no info_val, no gpu_addr, no atom */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), - /* no info_val, no gpu_addr, no atom */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), - /* info_val == GPU_IRQ_STATUS register */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), - /* info_val == bits cleared */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), - /* info_val == GPU_IRQ_STATUS register */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), - /* GPU addr==dump address */ - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), - KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), -/* - * Job Slot management events - */ - /* info_val==irq rawstat at start */ - KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), - /* info_val==jobs processed */ - KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), -/* In the following: - * - * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) - * - uatom==kernel-side mapped uatom address (for correlation with user-side) - */ - /* info_val==exit code; gpu_addr==chain gpuaddr */ - KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), - /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), - /* gpu_addr is as follows: - * - If JS_STATUS active after soft-stop, val==gpu addr written to - * JS_HEAD on submit - * - otherwise gpu_addr==0 */ - KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), - KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), - KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), - /* gpu_addr==JS_HEAD read */ - KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), - /* gpu_addr==JS_HEAD read */ - KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), - /* gpu_addr==JS_HEAD read */ - KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), - /* gpu_addr==JS_TAIL read */ - KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), -/* gpu_addr is as follows: - * - If JS_STATUS active before soft-stop, val==JS_HEAD - * - otherwise gpu_addr==0 - */ - /* gpu_addr==JS_HEAD read */ - KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), - KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), - KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), - /* info_val == is_scheduled */ - KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), - /* info_val == is_scheduled */ - KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), - KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), - /* info_val == nr jobs submitted */ - KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), - /* gpu_addr==JS_HEAD_NEXT last written */ - KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), - KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), - KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), - KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), -/* - * Job dispatch events - */ - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==0, info_val==0, uatom==0 */ - KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), -/* - * Scheduler Core events - */ - KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), - /* gpu_addr==last value written/would be written to JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), - KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), - KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), - /* info_val == lower 32 bits of affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), - /* info_val == lower 32 bits of affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), - /* info_val == the ctx attribute now on ctx */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), - /* info_val == the ctx attribute now on runpool */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), - /* info_val == the ctx attribute now off ctx */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), - /* info_val == the ctx attribute now off runpool */ - KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), -/* - * Scheduler Policy events - */ - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), - /* info_val == whether it was evicted */ - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), - /* gpu_addr==JS_HEAD to write if the job were run */ - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), - KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), -/* - * Power Management Events - */ - KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), - KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), - KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), - /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ - KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), - KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), - KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED), - KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED), - KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), - KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), - KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), - KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), - KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), - KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), - KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), - /* info_val == policy number, or -1 for "Already changing" */ - KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), - KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), - /* info_val == policy number */ - KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), - /* info_val == policy number */ - KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), -/* Unused code just to make it easier to not have a comma at the end. - * All other codes MUST come before this */ - KBASE_TRACE_CODE_MAKE_CODE(DUMMY) - -#if 0 /* Dummy section to avoid breaking formatting */ -}; -#endif - -/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/mali_kbase/tl/mali_kbase_tracepoints.c b/mali_kbase/tl/mali_kbase_tracepoints.c index b028ef8..e445a3a 100644 --- a/mali_kbase/tl/mali_kbase_tracepoints.c +++ b/mali_kbase/tl/mali_kbase_tracepoints.c @@ -66,12 +66,18 @@ enum tl_msg_id_obj { KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, KBASE_TL_EVENT_ATOM_SOFTJOB_START, KBASE_TL_EVENT_ATOM_SOFTJOB_END, + KBASE_TL_EVENT_ARB_GRANTED, + KBASE_TL_EVENT_ARB_STARTED, + KBASE_TL_EVENT_ARB_STOP_REQUESTED, + KBASE_TL_EVENT_ARB_STOPPED, KBASE_JD_GPU_SOFT_RESET, KBASE_TL_KBASE_NEW_DEVICE, KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG, KBASE_TL_KBASE_NEW_CTX, KBASE_TL_KBASE_DEL_CTX, + KBASE_TL_KBASE_CTX_ASSIGN_AS, + KBASE_TL_KBASE_CTX_UNASSIGN_AS, KBASE_TL_KBASE_NEW_KCPUQUEUE, KBASE_TL_KBASE_DEL_KCPUQUEUE, KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, @@ -270,14 +276,30 @@ enum tl_msg_id_aux { "atom soft job has completed", \ "@p", \ "atom") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_GRANTED, \ + "Arbiter has granted gpu access", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STARTED, \ + "Driver is running again and able to process jobs", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOP_REQUESTED, \ + "Arbiter has requested driver to stop using gpu", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOPPED, \ + "Driver has stopped using gpu", \ + "@p", \ + "gpu") \ TRACEPOINT_DESC(KBASE_JD_GPU_SOFT_RESET, \ "gpu soft reset", \ "@p", \ "gpu") \ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \ "New KBase Device", \ - "@III", \ - "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs") \ + "@IIII", \ + "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs,kbase_device_as_count") \ TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, \ "CSG is programmed to a slot", \ "@III", \ @@ -294,6 +316,14 @@ enum tl_msg_id_aux { "Delete KBase Context", \ "@I", \ "kernel_ctx_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_ASSIGN_AS, \ + "Address Space is assigned to a KBase context", \ + "@II", \ + "kernel_ctx_id,kbase_device_as_index") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_UNASSIGN_AS, \ + "Address Space is unassigned from a KBase context", \ + "@I", \ + "kernel_ctx_id") \ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_KCPUQUEUE, \ "New KCPU Queue", \ "@pII", \ @@ -1463,6 +1493,94 @@ void __kbase_tlstream_tl_event_atom_softjob_end( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_event_arb_granted( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_EVENT_ARB_GRANTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_arb_started( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_EVENT_ARB_STARTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_arb_stop_requested( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_EVENT_ARB_STOP_REQUESTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_arb_stopped( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_EVENT_ARB_STOPPED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_jd_gpu_soft_reset( struct kbase_tlstream *stream, const void *gpu) @@ -1757,13 +1875,15 @@ void __kbase_tlstream_tl_kbase_new_device( struct kbase_tlstream *stream, u32 kbase_device_id, u32 kbase_device_gpu_core_count, - u32 kbase_device_max_num_csgs) + u32 kbase_device_max_num_csgs, + u32 kbase_device_as_count) { const u32 msg_id = KBASE_TL_KBASE_NEW_DEVICE; const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(kbase_device_id) + sizeof(kbase_device_gpu_core_count) + sizeof(kbase_device_max_num_csgs) + + sizeof(kbase_device_as_count) ; char *buffer; unsigned long acq_flags; @@ -1779,6 +1899,8 @@ void __kbase_tlstream_tl_kbase_new_device( pos, &kbase_device_gpu_core_count, sizeof(kbase_device_gpu_core_count)); pos = kbasep_serialize_bytes(buffer, pos, &kbase_device_max_num_csgs, sizeof(kbase_device_max_num_csgs)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_as_count, sizeof(kbase_device_as_count)); kbase_tlstream_msgbuf_release(stream, acq_flags); } @@ -1887,6 +2009,54 @@ void __kbase_tlstream_tl_kbase_del_ctx( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_kbase_ctx_assign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_as_index) +{ + const u32 msg_id = KBASE_TL_KBASE_CTX_ASSIGN_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + + sizeof(kbase_device_as_index) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_as_index, sizeof(kbase_device_as_index)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_ctx_unassign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id) +{ + const u32 msg_id = KBASE_TL_KBASE_CTX_UNASSIGN_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_tl_kbase_new_kcpuqueue( struct kbase_tlstream *stream, const void *kcpu_queue, diff --git a/mali_kbase/tl/mali_kbase_tracepoints.h b/mali_kbase/tl/mali_kbase_tracepoints.h index fa2c399..7ea8ba8 100644 --- a/mali_kbase/tl/mali_kbase_tracepoints.h +++ b/mali_kbase/tl/mali_kbase_tracepoints.h @@ -56,8 +56,8 @@ extern const size_t __aux_stream_offset; * relies on offset variables: * __obj_stream_offset and __aux_stream_offset. */ -#define __TL_DISPATCH_STREAM(kbdev, stype) \ - ((struct kbase_tlstream *) \ +#define __TL_DISPATCH_STREAM(kbdev, stype) \ + ((struct kbase_tlstream *) \ ((u8 *)kbdev->timeline + __ ## stype ## _stream_offset)) struct tp_desc; @@ -226,6 +226,18 @@ void __kbase_tlstream_tl_event_atom_softjob_start( void __kbase_tlstream_tl_event_atom_softjob_end( struct kbase_tlstream *stream, const void *atom); +void __kbase_tlstream_tl_event_arb_granted( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_event_arb_started( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_event_arb_stop_requested( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_event_arb_stopped( + struct kbase_tlstream *stream, + const void *gpu); void __kbase_tlstream_jd_gpu_soft_reset( struct kbase_tlstream *stream, const void *gpu); @@ -275,7 +287,8 @@ void __kbase_tlstream_tl_kbase_new_device( struct kbase_tlstream *stream, u32 kbase_device_id, u32 kbase_device_gpu_core_count, - u32 kbase_device_max_num_csgs); + u32 kbase_device_max_num_csgs, + u32 kbase_device_as_count); void __kbase_tlstream_tl_kbase_device_program_csg( struct kbase_tlstream *stream, u32 kbase_device_id, @@ -292,6 +305,13 @@ void __kbase_tlstream_tl_kbase_new_ctx( void __kbase_tlstream_tl_kbase_del_ctx( struct kbase_tlstream *stream, u32 kernel_ctx_id); +void __kbase_tlstream_tl_kbase_ctx_assign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_as_index); +void __kbase_tlstream_tl_kbase_ctx_unassign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id); void __kbase_tlstream_tl_kbase_new_kcpuqueue( struct kbase_tlstream *stream, const void *kcpu_queue, @@ -465,10 +485,10 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NEW_CTX - * object ctx is created * - * @kbdev: Kbase device - * @ctx: Name of the context object - * @ctx_nr: Kernel context number - * @tgid: Thread Group Id + * @kbdev: Kbase device + * @ctx: Name of the context object + * @ctx_nr: Kernel context number + * @tgid: Thread Group Id */ #define KBASE_TLSTREAM_TL_NEW_CTX( \ kbdev, \ @@ -477,10 +497,10 @@ struct kbase_tlstream; tgid \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ ctx, ctx_nr, tgid); \ } while (0) @@ -488,10 +508,10 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NEW_GPU - * object gpu is created * - * @kbdev: Kbase device - * @gpu: Name of the GPU object - * @gpu_id: Name of the GPU object - * @core_count: Number of cores this GPU hosts + * @kbdev: Kbase device + * @gpu: Name of the GPU object + * @gpu_id: Name of the GPU object + * @core_count: Number of cores this GPU hosts */ #define KBASE_TLSTREAM_TL_NEW_GPU( \ kbdev, \ @@ -500,10 +520,10 @@ struct kbase_tlstream; core_count \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_gpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ gpu, gpu_id, core_count); \ } while (0) @@ -511,10 +531,10 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NEW_LPU - * object lpu is created * - * @kbdev: Kbase device - * @lpu: Name of the Logical Processing Unit object - * @lpu_nr: Sequential number assigned to the newly created LPU - * @lpu_fn: Property describing functional abilities of this LPU + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object + * @lpu_nr: Sequential number assigned to the newly created LPU + * @lpu_fn: Property describing functional abilities of this LPU */ #define KBASE_TLSTREAM_TL_NEW_LPU( \ kbdev, \ @@ -523,10 +543,10 @@ struct kbase_tlstream; lpu_fn \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_lpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ lpu, lpu_nr, lpu_fn); \ } while (0) @@ -534,9 +554,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NEW_ATOM - * object atom is created * - * @kbdev: Kbase device - * @atom: Atom identifier - * @atom_nr: Sequential number of an atom + * @kbdev: Kbase device + * @atom: Atom identifier + * @atom_nr: Sequential number of an atom */ #define KBASE_TLSTREAM_TL_NEW_ATOM( \ kbdev, \ @@ -544,10 +564,10 @@ struct kbase_tlstream; atom_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_atom( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, atom_nr); \ } while (0) @@ -555,9 +575,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NEW_AS - * address space object is created * - * @kbdev: Kbase device - * @address_space: Name of the address space object - * @as_nr: Address space number + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @as_nr: Address space number */ #define KBASE_TLSTREAM_TL_NEW_AS( \ kbdev, \ @@ -565,10 +585,10 @@ struct kbase_tlstream; as_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_as( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ address_space, as_nr); \ } while (0) @@ -576,18 +596,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_DEL_CTX - * context is destroyed * - * @kbdev: Kbase device - * @ctx: Name of the context object + * @kbdev: Kbase device + * @ctx: Name of the context object */ #define KBASE_TLSTREAM_TL_DEL_CTX( \ kbdev, \ ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_del_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ ctx); \ } while (0) @@ -595,18 +615,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_DEL_ATOM - * atom is destroyed * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_DEL_ATOM( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_del_atom( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom); \ } while (0) @@ -614,9 +634,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU - * lpu is deleted with gpu * - * @kbdev: Kbase device - * @lpu: Name of the Logical Processing Unit object - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU( \ kbdev, \ @@ -624,10 +644,10 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_lifelink_lpu_gpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ lpu, gpu); \ } while (0) @@ -635,9 +655,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_LIFELINK_AS_GPU - * address space is deleted with gpu * - * @kbdev: Kbase device - * @address_space: Name of the address space object - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_TL_LIFELINK_AS_GPU( \ kbdev, \ @@ -645,10 +665,10 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_lifelink_as_gpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ address_space, gpu); \ } while (0) @@ -656,9 +676,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_RET_CTX_LPU - * context is retained by lpu * - * @kbdev: Kbase device - * @ctx: Name of the context object - * @lpu: Name of the Logical Processing Unit object + * @kbdev: Kbase device + * @ctx: Name of the context object + * @lpu: Name of the Logical Processing Unit object */ #define KBASE_TLSTREAM_TL_RET_CTX_LPU( \ kbdev, \ @@ -666,10 +686,10 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_ctx_lpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ ctx, lpu); \ } while (0) @@ -677,9 +697,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_RET_ATOM_CTX - * atom is retained by context * - * @kbdev: Kbase device - * @atom: Atom identifier - * @ctx: Name of the context object + * @kbdev: Kbase device + * @atom: Atom identifier + * @ctx: Name of the context object */ #define KBASE_TLSTREAM_TL_RET_ATOM_CTX( \ kbdev, \ @@ -687,10 +707,10 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, ctx); \ } while (0) @@ -698,10 +718,10 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_RET_ATOM_LPU - * atom is retained by lpu * - * @kbdev: Kbase device - * @atom: Atom identifier - * @lpu: Name of the Logical Processing Unit object - * @attrib_match_list: List containing match operator attributes + * @kbdev: Kbase device + * @atom: Atom identifier + * @lpu: Name of the Logical Processing Unit object + * @attrib_match_list: List containing match operator attributes */ #define KBASE_TLSTREAM_TL_RET_ATOM_LPU( \ kbdev, \ @@ -710,10 +730,10 @@ struct kbase_tlstream; attrib_match_list \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_lpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, lpu, attrib_match_list); \ } while (0) @@ -721,9 +741,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NRET_CTX_LPU - * context is released by lpu * - * @kbdev: Kbase device - * @ctx: Name of the context object - * @lpu: Name of the Logical Processing Unit object + * @kbdev: Kbase device + * @ctx: Name of the context object + * @lpu: Name of the Logical Processing Unit object */ #define KBASE_TLSTREAM_TL_NRET_CTX_LPU( \ kbdev, \ @@ -731,10 +751,10 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_ctx_lpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ ctx, lpu); \ } while (0) @@ -742,9 +762,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - * atom is released by context * - * @kbdev: Kbase device - * @atom: Atom identifier - * @ctx: Name of the context object + * @kbdev: Kbase device + * @atom: Atom identifier + * @ctx: Name of the context object */ #define KBASE_TLSTREAM_TL_NRET_ATOM_CTX( \ kbdev, \ @@ -752,10 +772,10 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, ctx); \ } while (0) @@ -763,9 +783,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - * atom is released by lpu * - * @kbdev: Kbase device - * @atom: Atom identifier - * @lpu: Name of the Logical Processing Unit object + * @kbdev: Kbase device + * @atom: Atom identifier + * @lpu: Name of the Logical Processing Unit object */ #define KBASE_TLSTREAM_TL_NRET_ATOM_LPU( \ kbdev, \ @@ -773,10 +793,10 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_lpu( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, lpu); \ } while (0) @@ -784,9 +804,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_RET_AS_CTX - * address space is retained by context * - * @kbdev: Kbase device - * @address_space: Name of the address space object - * @ctx: Name of the context object + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @ctx: Name of the context object */ #define KBASE_TLSTREAM_TL_RET_AS_CTX( \ kbdev, \ @@ -794,10 +814,10 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_as_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ address_space, ctx); \ } while (0) @@ -805,9 +825,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NRET_AS_CTX - * address space is released by context * - * @kbdev: Kbase device - * @address_space: Name of the address space object - * @ctx: Name of the context object + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @ctx: Name of the context object */ #define KBASE_TLSTREAM_TL_NRET_AS_CTX( \ kbdev, \ @@ -815,10 +835,10 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_as_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ address_space, ctx); \ } while (0) @@ -826,9 +846,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_RET_ATOM_AS - * atom is retained by address space * - * @kbdev: Kbase device - * @atom: Atom identifier - * @address_space: Name of the address space object + * @kbdev: Kbase device + * @atom: Atom identifier + * @address_space: Name of the address space object */ #define KBASE_TLSTREAM_TL_RET_ATOM_AS( \ kbdev, \ @@ -836,10 +856,10 @@ struct kbase_tlstream; address_space \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_as( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, address_space); \ } while (0) @@ -847,9 +867,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_NRET_ATOM_AS - * atom is released by address space * - * @kbdev: Kbase device - * @atom: Atom identifier - * @address_space: Name of the address space object + * @kbdev: Kbase device + * @atom: Atom identifier + * @address_space: Name of the address space object */ #define KBASE_TLSTREAM_TL_NRET_ATOM_AS( \ kbdev, \ @@ -857,10 +877,10 @@ struct kbase_tlstream; address_space \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_as( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, address_space); \ } while (0) @@ -868,11 +888,11 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - * atom job slot attributes * - * @kbdev: Kbase device - * @atom: Atom identifier - * @descriptor: Job descriptor address - * @affinity: Job affinity - * @config: Job config + * @kbdev: Kbase device + * @atom: Atom identifier + * @descriptor: Job descriptor address + * @affinity: Job affinity + * @config: Job config */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG( \ kbdev, \ @@ -882,10 +902,10 @@ struct kbase_tlstream; config \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_config( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, descriptor, affinity, config); \ } while (0) @@ -893,9 +913,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - * atom priority * - * @kbdev: Kbase device - * @atom: Atom identifier - * @prio: Atom priority + * @kbdev: Kbase device + * @atom: Atom identifier + * @prio: Atom priority */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY( \ kbdev, \ @@ -903,7 +923,7 @@ struct kbase_tlstream; prio \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_priority( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -914,9 +934,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - * atom state * - * @kbdev: Kbase device - * @atom: Atom identifier - * @state: Atom state + * @kbdev: Kbase device + * @atom: Atom identifier + * @state: Atom state */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE( \ kbdev, \ @@ -924,7 +944,7 @@ struct kbase_tlstream; state \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_state( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -935,15 +955,15 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED - * atom caused priority change * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_prioritized( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -954,20 +974,20 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - * jit done for atom * - * @kbdev: Kbase device - * @atom: Atom identifier - * @edit_addr: Address edited by jit - * @new_addr: Address placed into the edited location - * @jit_flags: Flags specifying the special requirements for + * @kbdev: Kbase device + * @atom: Atom identifier + * @edit_addr: Address edited by jit + * @new_addr: Address placed into the edited location + * @jit_flags: Flags specifying the special requirements for * the JIT allocation. - * @mem_flags: Flags defining the properties of a memory region - * @j_id: Unique ID provided by the caller, this is used + * @mem_flags: Flags defining the properties of a memory region + * @j_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. - * @com_pgs: The minimum number of physical pages which + * @com_pgs: The minimum number of physical pages which * should back the allocation. - * @extent: Granularity of physical pages to grow the + * @extent: Granularity of physical pages to grow the * allocation by during a fault. - * @va_pgs: The minimum number of virtual pages required + * @va_pgs: The minimum number of virtual pages required */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \ kbdev, \ @@ -982,7 +1002,7 @@ struct kbase_tlstream; va_pgs \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jit( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -993,9 +1013,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_JIT_USEDPAGES - * used pages for jit * - * @kbdev: Kbase device - * @used_pages: Number of pages used for jit - * @j_id: Unique ID provided by the caller, this is used + * @kbdev: Kbase device + * @used_pages: Number of pages used for jit + * @j_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. */ #define KBASE_TLSTREAM_TL_JIT_USEDPAGES( \ @@ -1004,10 +1024,10 @@ struct kbase_tlstream; j_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jit_usedpages( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ used_pages, j_id); \ } while (0) @@ -1015,22 +1035,22 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO - * Information about JIT allocations * - * @kbdev: Kbase device - * @atom: Atom identifier - * @va_pgs: The minimum number of virtual pages required - * @com_pgs: The minimum number of physical pages which + * @kbdev: Kbase device + * @atom: Atom identifier + * @va_pgs: The minimum number of virtual pages required + * @com_pgs: The minimum number of physical pages which * should back the allocation. - * @extent: Granularity of physical pages to grow the + * @extent: Granularity of physical pages to grow the * allocation by during a fault. - * @j_id: Unique ID provided by the caller, this is used + * @j_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. - * @bin_id: The JIT allocation bin, used in conjunction with + * @bin_id: The JIT allocation bin, used in conjunction with * max_allocations to limit the number of each * type of JIT allocation. - * @max_allocs: Maximum allocations allowed in this bin. - * @jit_flags: Flags specifying the special requirements for + * @max_allocs: Maximum allocations allowed in this bin. + * @jit_flags: Flags specifying the special requirements for * the JIT allocation. - * @usg_id: A hint about which allocation should be reused. + * @usg_id: A hint about which allocation should be reused. */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( \ kbdev, \ @@ -1045,10 +1065,10 @@ struct kbase_tlstream; usg_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jitallocinfo( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, va_pgs, com_pgs, extent, j_id, bin_id, max_allocs, jit_flags, usg_id); \ } while (0) @@ -1056,9 +1076,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO - * Information about JIT frees * - * @kbdev: Kbase device - * @atom: Atom identifier - * @j_id: Unique ID provided by the caller, this is used + * @kbdev: Kbase device + * @atom: Atom identifier + * @j_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. */ #define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO( \ @@ -1067,10 +1087,10 @@ struct kbase_tlstream; j_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jitfreeinfo( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom, j_id); \ } while (0) @@ -1078,11 +1098,11 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - * address space attributes * - * @kbdev: Kbase device - * @address_space: Name of the address space object - * @transtab: Configuration of the TRANSTAB register - * @memattr: Configuration of the MEMATTR register - * @transcfg: Configuration of the TRANSCFG register (or zero if not present) + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @transtab: Configuration of the TRANSTAB register + * @memattr: Configuration of the MEMATTR register + * @transcfg: Configuration of the TRANSCFG register (or zero if not present) */ #define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG( \ kbdev, \ @@ -1092,10 +1112,10 @@ struct kbase_tlstream; transcfg \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_as_config( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ address_space, transtab, memattr, transcfg); \ } while (0) @@ -1103,18 +1123,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP - * softstop event on given lpu * - * @kbdev: Kbase device - * @lpu: Name of the Logical Processing Unit object + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object */ #define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( \ kbdev, \ lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_lpu_softstop( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ lpu); \ } while (0) @@ -1122,18 +1142,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX - * atom softstopped * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softstop_ex( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom); \ } while (0) @@ -1141,18 +1161,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE - * atom softstop issued * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softstop_issue( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom); \ } while (0) @@ -1160,18 +1180,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START - * atom soft job has started * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softjob_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom); \ } while (0) @@ -1179,37 +1199,113 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END - * atom soft job has completed * - * @kbdev: Kbase device - * @atom: Atom identifier + * @kbdev: Kbase device + * @atom: Atom identifier */ #define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END( \ kbdev, \ atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softjob_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ atom); \ } while (0) /** + * KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED - + * Arbiter has granted gpu access + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_arb_granted( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ARB_STARTED - + * Driver is running again and able to process jobs + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_ARB_STARTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_arb_started( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED - + * Arbiter has requested driver to stop using gpu + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_arb_stop_requested( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED - + * Driver has stopped using gpu + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_arb_stopped( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - * gpu soft reset * - * @kbdev: Kbase device - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_JD_GPU_SOFT_RESET( \ kbdev, \ gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_jd_gpu_soft_reset( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ + __TL_DISPATCH_STREAM(kbdev, obj), \ gpu); \ } while (0) @@ -1217,9 +1313,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PM_STATE - * PM state * - * @kbdev: Kbase device - * @core_type: Core type (shader, tiler, l2 cache, l3 cache) - * @core_state_bitset: 64bits bitmask reporting power state of the cores + * @kbdev: Kbase device + * @core_type: Core type (shader, tiler, l2 cache, l3 cache) + * @core_state_bitset: 64bits bitmask reporting power state of the cores * (1-ON, 0-OFF) */ #define KBASE_TLSTREAM_AUX_PM_STATE( \ @@ -1228,10 +1324,10 @@ struct kbase_tlstream; core_state_bitset \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pm_state( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ core_type, core_state_bitset); \ } while (0) @@ -1239,10 +1335,10 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PAGEFAULT - * Page fault * - * @kbdev: Kbase device - * @ctx_nr: Kernel context number - * @as_nr: Address space number - * @page_cnt_change: Number of pages to be added + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @as_nr: Address space number + * @page_cnt_change: Number of pages to be added */ #define KBASE_TLSTREAM_AUX_PAGEFAULT( \ kbdev, \ @@ -1251,10 +1347,10 @@ struct kbase_tlstream; page_cnt_change \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pagefault( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ ctx_nr, as_nr, page_cnt_change); \ } while (0) @@ -1262,9 +1358,9 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PAGESALLOC - * Total alloc pages change * - * @kbdev: Kbase device - * @ctx_nr: Kernel context number - * @page_cnt: Number of pages used by the context + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @page_cnt: Number of pages used by the context */ #define KBASE_TLSTREAM_AUX_PAGESALLOC( \ kbdev, \ @@ -1272,10 +1368,10 @@ struct kbase_tlstream; page_cnt \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pagesalloc( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ ctx_nr, page_cnt); \ } while (0) @@ -1283,18 +1379,18 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - * New device frequency target * - * @kbdev: Kbase device - * @target_freq: New target frequency + * @kbdev: Kbase device + * @target_freq: New target frequency */ #define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET( \ kbdev, \ target_freq \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_devfreq_target( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ target_freq); \ } while (0) @@ -1302,15 +1398,15 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - * enter protected mode start * - * @kbdev: Kbase device - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START( \ kbdev, \ gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_enter_start( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -1321,15 +1417,15 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - * enter protected mode end * - * @kbdev: Kbase device - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END( \ kbdev, \ gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_enter_end( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -1340,15 +1436,15 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - * leave protected mode start * - * @kbdev: Kbase device - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START( \ kbdev, \ gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_leave_start( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -1359,15 +1455,15 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - * leave protected mode end * - * @kbdev: Kbase device - * @gpu: Name of the GPU object + * @kbdev: Kbase device + * @gpu: Name of the GPU object */ #define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END( \ kbdev, \ gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_leave_end( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -1378,13 +1474,13 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_JIT_STATS - * per-bin JIT statistics * - * @kbdev: Kbase device - * @ctx_nr: Kernel context number - * @bid: JIT bin id - * @max_allocs: Maximum allocations allowed in this bin. - * @allocs: Number of active allocations in this bin - * @va_pages: Number of virtual pages allocated in this bin - * @ph_pages: Number of physical pages allocated in this bin + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @bid: JIT bin id + * @max_allocs: Maximum allocations allowed in this bin. + * @allocs: Number of active allocations in this bin + * @va_pages: Number of virtual pages allocated in this bin + * @ph_pages: Number of physical pages allocated in this bin */ #define KBASE_TLSTREAM_AUX_JIT_STATS( \ kbdev, \ @@ -1396,10 +1492,10 @@ struct kbase_tlstream; ph_pages \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_jit_stats( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ ctx_nr, bid, max_allocs, allocs, va_pages, ph_pages); \ } while (0) @@ -1407,11 +1503,11 @@ struct kbase_tlstream; * KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT - * event on a given job slot * - * @kbdev: Kbase device - * @ctx: Name of the context object - * @slot_nr: Job slot number - * @atom_nr: Sequential number of an atom - * @event: Event type. One of TL_JS_EVENT values + * @kbdev: Kbase device + * @ctx: Name of the context object + * @slot_nr: Job slot number + * @atom_nr: Sequential number of an atom + * @event: Event type. One of TL_JS_EVENT values */ #define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( \ kbdev, \ @@ -1421,10 +1517,10 @@ struct kbase_tlstream; event \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_event_job_slot( \ - __TL_DISPATCH_STREAM(kbdev, aux), \ + __TL_DISPATCH_STREAM(kbdev, aux), \ ctx, slot_nr, atom_nr, event); \ } while (0) @@ -1432,33 +1528,29 @@ struct kbase_tlstream; * KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE - * New KBase Device * - * @kbdev: Kbase device - * @kbase_device_id: The id of the physical hardware - * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware - * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware + * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports + * @kbase_device_as_count: The number of address spaces the physical hardware has available */ #define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ kbdev, \ kbase_device_id, \ kbase_device_gpu_core_count, \ - kbase_device_max_num_csgs \ + kbase_device_max_num_csgs, \ + kbase_device_as_count \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_new_device( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kbase_device_id, kbase_device_gpu_core_count, kbase_device_max_num_csgs); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - * CSG is programmed to a slot * - * @kbdev: Kbase device - * @kbase_device_id: The id of the physical hardware - * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace - * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace + * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed */ #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ kbdev, \ @@ -1466,83 +1558,87 @@ struct kbase_tlstream; gpu_cmdq_grp_handle, \ kbase_device_csg_slot_index \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_device_program_csg( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kbase_device_id, gpu_cmdq_grp_handle, kbase_device_csg_slot_index); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - * CSG is deprogrammed from a slot * - * @kbdev: Kbase device - * @kbase_device_id: The id of the physical hardware - * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed */ #define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ kbdev, \ kbase_device_id, \ kbase_device_csg_slot_index \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_device_deprogram_csg( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kbase_device_id, kbase_device_csg_slot_index); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - * New KBase Context * - * @kbdev: Kbase device - * @kernel_ctx_id: Unique ID for the KBase Context - * @kbase_device_id: The id of the physical hardware + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + * @kbase_device_id: The id of the physical hardware */ #define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ kbdev, \ kernel_ctx_id, \ kbase_device_id \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_new_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kernel_ctx_id, kbase_device_id); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - * Delete KBase Context * - * @kbdev: Kbase device - * @kernel_ctx_id: Unique ID for the KBase Context + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context */ #define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ kbdev, \ kernel_ctx_id \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_del_ctx( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kernel_ctx_id); \ - } while (0) + do { } while (0) + +/** + * KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS - + * Address Space is assigned to a KBase context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + * @kbase_device_as_index: The index of the device address space being assigned + */ +#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ + kbdev, \ + kernel_ctx_id, \ + kbase_device_as_index \ + ) \ + do { } while (0) + +/** + * KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS - + * Address Space is unassigned from a KBase context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + */ +#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ + kbdev, \ + kernel_ctx_id \ + ) \ + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - * New KCPU Queue * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @kernel_ctx_id: Unique ID for the KBase Context - * @kcpuq_num_pending_cmds: Number of commands already enqueued + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @kernel_ctx_id: Unique ID for the KBase Context + * @kcpuq_num_pending_cmds: Number of commands already enqueued * in the KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ @@ -1551,102 +1647,72 @@ struct kbase_tlstream; kernel_ctx_id, \ kcpuq_num_pending_cmds \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_new_kcpuqueue( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, kernel_ctx_id, kcpuq_num_pending_cmds); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - * Delete KCPU Queue * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_del_kcpuqueue( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - * KCPU Queue enqueues Signal on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @fence: Fence object handle + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @fence: Fence object handle */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ kbdev, \ kcpu_queue, \ fence \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, fence); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - * KCPU Queue enqueues Wait on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @fence: Fence object handle + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @fence: Fence object handle */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ kbdev, \ kcpu_queue, \ fence \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, fence); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT - * Begin array of KCPU Queue enqueues Wait on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_wait( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT - * Array item of KCPU Queue enqueues Wait on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @cqs_obj_gpu_addr: CQS Object GPU ptr - * @cqs_obj_compare_value: Semaphore value that should be exceeded + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @cqs_obj_gpu_addr: CQS Object GPU ptr + * @cqs_obj_compare_value: Semaphore value that should be exceeded * for the WAIT to pass */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ @@ -1655,256 +1721,184 @@ struct kbase_tlstream; cqs_obj_gpu_addr, \ cqs_obj_compare_value \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_wait( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, cqs_obj_gpu_addr, cqs_obj_compare_value); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT - * End array of KCPU Queue enqueues Wait on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_wait( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET - * Begin array of KCPU Queue enqueues Set on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_set( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET - * Array item of KCPU Queue enqueues Set on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @cqs_obj_gpu_addr: CQS Object GPU ptr + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @cqs_obj_gpu_addr: CQS Object GPU ptr */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET( \ kbdev, \ kcpu_queue, \ cqs_obj_gpu_addr \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_set( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, cqs_obj_gpu_addr); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET - * End array of KCPU Queue enqueues Set on Cross Queue Sync Object * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_set( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY - * Begin array of KCPU Queue enqueues Debug Copy * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_debugcopy( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY - * Array item of KCPU Queue enqueues Debug Copy * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @debugcopy_dst_size: Debug Copy destination size + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @debugcopy_dst_size: Debug Copy destination size */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ kbdev, \ kcpu_queue, \ debugcopy_dst_size \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_debugcopy( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, debugcopy_dst_size); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY - * End array of KCPU Queue enqueues Debug Copy * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_debugcopy( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - * KCPU Queue enqueues Map Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @map_import_buf_gpu_addr: Map import buffer GPU ptr + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ kbdev, \ kcpu_queue, \ map_import_buf_gpu_addr \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, map_import_buf_gpu_addr); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - * KCPU Queue enqueues Unmap Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @map_import_buf_gpu_addr: Map import buffer GPU ptr + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ kbdev, \ kcpu_queue, \ map_import_buf_gpu_addr \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, map_import_buf_gpu_addr); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - * KCPU Queue enqueues Unmap Import ignoring reference count * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @map_import_buf_gpu_addr: Map import buffer GPU ptr + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ kbdev, \ kcpu_queue, \ map_import_buf_gpu_addr \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, map_import_buf_gpu_addr); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - * Begin array of KCPU Queue enqueues JIT Alloc * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - * Array item of KCPU Queue enqueues JIT Alloc * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write * the JIT allocated GPU virtual address to - * @jit_alloc_va_pages: The minimum number of virtual pages required - * @jit_alloc_commit_pages: The minimum number of physical pages which + * @jit_alloc_va_pages: The minimum number of virtual pages required + * @jit_alloc_commit_pages: The minimum number of physical pages which * should back the allocation - * @jit_alloc_extent: Granularity of physical pages to grow the allocation + * @jit_alloc_extent: Granularity of physical pages to grow the allocation * by during a fault - * @jit_alloc_jit_id: Unique ID provided by the caller, this is used + * @jit_alloc_jit_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. Zero is not a valid value - * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with + * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with * max_allocations to limit the number of each type of JIT allocation - * @jit_alloc_max_allocations: The maximum number of allocations + * @jit_alloc_max_allocations: The maximum number of allocations * allowed within the bin specified by bin_id. Should be the same for all * JIT allocations within the same bin. - * @jit_alloc_flags: Flags specifying the special requirements for the + * @jit_alloc_flags: Flags specifying the special requirements for the * JIT allocation - * @jit_alloc_usage_id: A hint about which allocation should be + * @jit_alloc_usage_id: A hint about which allocation should be * reused. The kernel should attempt to use a previous allocation with the same * usage_id */ @@ -1921,59 +1915,41 @@ struct kbase_tlstream; jit_alloc_flags, \ jit_alloc_usage_id \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, jit_alloc_gpu_alloc_addr_dest, jit_alloc_va_pages, jit_alloc_commit_pages, jit_alloc_extent, jit_alloc_jit_id, jit_alloc_bin_id, jit_alloc_max_allocations, jit_alloc_flags, jit_alloc_usage_id); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - * End array of KCPU Queue enqueues JIT Alloc * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - * Begin array of KCPU Queue enqueues JIT Free * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - * Array item of KCPU Queue enqueues JIT Free * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @jit_alloc_jit_id: Unique ID provided by the caller, this is used + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_jit_id: Unique ID provided by the caller, this is used * to pair allocation and free requests. Zero is not a valid value */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ @@ -1981,364 +1957,250 @@ struct kbase_tlstream; kcpu_queue, \ jit_alloc_jit_id \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, jit_alloc_jit_id); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - * End array of KCPU Queue enqueues JIT Free * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - * KCPU Queue starts a Signal on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - * KCPU Queue ends a Signal on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - * KCPU Queue starts a Wait on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - * KCPU Queue ends a Wait on Fence * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - * KCPU Queue starts a Wait on an array of Cross Queue Sync Objects * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - * KCPU Queue ends a Wait on an array of Cross Queue Sync Objects * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - * KCPU Queue executes a Set on an array of Cross Queue Sync Objects * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START - * KCPU Queue starts an array of Debug Copys * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END - * KCPU Queue ends an array of Debug Copys * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - * KCPU Queue starts a Map Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - * KCPU Queue ends a Map Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - * KCPU Queue starts an Unmap Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - * KCPU Queue ends an Unmap Import * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - * KCPU Queue starts an Unmap Import ignoring reference count * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - * KCPU Queue ends an Unmap Import ignoring reference count * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - * KCPU Queue starts an array of JIT Allocs * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - * Begin array of KCPU Queue ends an array of JIT Allocs * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - * Array item of KCPU Queue ends an array of JIT Allocs * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address - * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address + * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ @@ -2346,78 +2208,54 @@ struct kbase_tlstream; jit_alloc_gpu_alloc_addr, \ jit_alloc_mmu_flags \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, jit_alloc_gpu_alloc_addr, jit_alloc_mmu_flags); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - * End array of KCPU Queue ends an array of JIT Allocs * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - * KCPU Queue starts an array of JIT Frees * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - * Begin array of KCPU Queue ends an array of JIT Frees * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - * Array item of KCPU Queue ends an array of JIT Frees * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - * @jit_free_pages_used: The actual number of pages used by the JIT + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_free_pages_used: The actual number of pages used by the JIT * allocation */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ @@ -2425,72 +2263,48 @@ struct kbase_tlstream; kcpu_queue, \ jit_free_pages_used \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue, jit_free_pages_used); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - * End array of KCPU Queue ends an array of JIT Frees * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER - * KCPU Queue executes an Error Barrier * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue */ #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ kbdev, \ kcpu_queue \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & TLSTREAM_ENABLED) \ - __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - kcpu_queue); \ - } while (0) + do { } while (0) /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - * An overflow has happened with the CSFFW Timeline stream * - * @kbdev: Kbase device - * @csffw_timestamp: Timestamp of a CSFFW event - * @csffw_cycle: Cycle number of a CSFFW event + * @kbdev: Kbase device + * @csffw_timestamp: Timestamp of a CSFFW event + * @csffw_cycle: Cycle number of a CSFFW event */ #define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ kbdev, \ csffw_timestamp, \ csffw_cycle \ ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ - __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - csffw_timestamp, csffw_cycle); \ - } while (0) + do { } while (0) /* Gator tracepoints are hooked into TLSTREAM interface. @@ -2504,10 +2318,10 @@ struct kbase_tlstream; * with corresponding GATOR_JOB_SLOT values. */ #undef KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT -#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \ +#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \ context, slot_nr, atom_nr, event) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_job_slots_event(kbdev->id, \ GATOR_MAKE_EVENT(event, slot_nr), \ context, (u8) atom_nr); \ @@ -2520,7 +2334,7 @@ struct kbase_tlstream; #undef KBASE_TLSTREAM_AUX_PM_STATE #define KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_pm_status(kbdev->id, \ core_type, state); \ if (enabled & TLSTREAM_ENABLED) \ @@ -2530,10 +2344,10 @@ struct kbase_tlstream; } while (0) #undef KBASE_TLSTREAM_AUX_PAGEFAULT -#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \ +#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \ ctx_nr, as_nr, page_cnt_change) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + int enabled = atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_page_fault_insert_pages(kbdev->id, \ as_nr, \ page_cnt_change); \ @@ -2550,8 +2364,8 @@ struct kbase_tlstream; #undef KBASE_TLSTREAM_AUX_PAGESALLOC #define KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, ctx_nr, page_cnt) \ do { \ - int enabled = atomic_read(&kbdev->timeline_is_enabled); \ - u32 global_pages_count = \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 global_pages_count = \ atomic_read(&kbdev->memdev.used_pages); \ \ kbase_trace_mali_total_alloc_pages_change(kbdev->id, \ |