summaryrefslogtreecommitdiff
path: root/mali_kbase
diff options
context:
space:
mode:
authorSidath Senanayake <sidaths@google.com>2020-06-18 09:26:13 +0200
committerSidath Senanayake <sidaths@google.com>2020-06-18 09:26:13 +0200
commitbc3c01e61c8ce9783a8ab091053905effcae12de (patch)
tree43f9f48736f7259d8cc996b11eed003dc80d5bef /mali_kbase
parentb64f568f943e567534694cc993270adca96dcd06 (diff)
downloadgpu-bc3c01e61c8ce9783a8ab091053905effcae12de.tar.gz
Mali Valhall DDK r25p0 KMD
Provenance: 395644cb0 (collaborate/EAC/v_r25p0) VX504X08X-BU-00000-r25p0-01eac0 - Android DDK VX504X08X-BU-60000-r25p0-01eac0 - Android Document Bundle Signed-off-by: Sidath Senanayake <sidaths@google.com> Change-Id: I2cffddb42a554696d45b7f65c7bae8827a71341f
Diffstat (limited to 'mali_kbase')
-rw-r--r--mali_kbase/Kbuild16
-rw-r--r--mali_kbase/Kconfig21
-rw-r--r--mali_kbase/Mconfig25
-rw-r--r--mali_kbase/arbiter/Kbuild24
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbif.c175
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbif.h133
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbiter_defs.h95
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbiter_interface.h181
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbiter_pm.c645
-rw-r--r--mali_kbase/arbiter/mali_kbase_arbiter_pm.h159
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_device_hw.c21
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_device_internal.h12
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c84
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_instr_backend.c6
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_as.c7
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_hw.c85
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_internal.h8
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_jm_rb.c33
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_js_backend.c5
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c18
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h15
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_backend.c109
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_ca.c2
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_driver.c199
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_internal.h27
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_metrics.c6
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_pm_policy.c22
-rw-r--r--mali_kbase/backend/gpu/mali_kbase_time.c10
-rw-r--r--mali_kbase/build.bp25
-rw-r--r--mali_kbase/context/mali_kbase_context.c12
-rw-r--r--mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h170
-rw-r--r--mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h75
-rw-r--r--mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c113
-rw-r--r--mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h362
-rw-r--r--mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h150
-rw-r--r--mali_kbase/debug/mali_kbase_debug_ktrace.c342
-rw-r--r--mali_kbase/debug/mali_kbase_debug_ktrace.h219
-rw-r--r--mali_kbase/debug/mali_kbase_debug_ktrace_codes.h158
-rw-r--r--mali_kbase/debug/mali_kbase_debug_ktrace_defs.h152
-rw-r--r--mali_kbase/debug/mali_kbase_debug_ktrace_internal.h89
-rw-r--r--mali_kbase/debug/mali_kbase_debug_linux_ktrace.h99
-rw-r--r--mali_kbase/device/backend/mali_kbase_device_jm.c14
-rw-r--r--mali_kbase/device/mali_kbase_device.c317
-rw-r--r--mali_kbase/gpu/mali_kbase_gpu_regmap.h12
-rw-r--r--mali_kbase/jm/mali_base_jm_kernel.h1
-rw-r--r--mali_kbase/jm/mali_kbase_jm_defs.h15
-rw-r--r--mali_kbase/jm/mali_kbase_jm_ioctl.h4
-rw-r--r--mali_kbase/jm/mali_kbase_js_defs.h (renamed from mali_kbase/mali_kbase_js_defs.h)1
-rw-r--r--mali_kbase/mali_base_hwconfig_issues.h32
-rw-r--r--mali_kbase/mali_kbase.h216
-rw-r--r--mali_kbase/mali_kbase_core_linux.c148
-rw-r--r--mali_kbase/mali_kbase_ctx_sched.c142
-rw-r--r--mali_kbase/mali_kbase_ctx_sched.h76
-rw-r--r--mali_kbase/mali_kbase_defs.h134
-rw-r--r--mali_kbase/mali_kbase_dummy_job_wa.c102
-rw-r--r--mali_kbase/mali_kbase_event.c5
-rw-r--r--mali_kbase/mali_kbase_gpuprops.c47
-rw-r--r--mali_kbase/mali_kbase_gpuprops.h38
-rw-r--r--mali_kbase/mali_kbase_hw.c10
-rw-r--r--mali_kbase/mali_kbase_hwaccess_gpuprops.h28
-rw-r--r--mali_kbase/mali_kbase_hwaccess_time.h4
-rw-r--r--mali_kbase/mali_kbase_hwcnt_backend_gpu.c7
-rw-r--r--mali_kbase/mali_kbase_jd.c58
-rw-r--r--mali_kbase/mali_kbase_js.c272
-rw-r--r--mali_kbase/mali_kbase_js.h93
-rw-r--r--mali_kbase/mali_kbase_js_ctx_attr.c10
-rw-r--r--mali_kbase/mali_kbase_mem.c327
-rw-r--r--mali_kbase/mali_kbase_mem.h24
-rw-r--r--mali_kbase/mali_kbase_pm.c150
-rw-r--r--mali_kbase/mali_kbase_pm.h88
-rw-r--r--mali_kbase/mali_kbase_softjobs.c47
-rw-r--r--mali_kbase/mali_kbase_vinstr.c5
-rw-r--r--mali_kbase/mali_linux_kbase_trace.h204
-rw-r--r--mali_kbase/mali_linux_trace.h75
-rw-r--r--mali_kbase/mmu/backend/mali_kbase_mmu_jm.c12
-rw-r--r--mali_kbase/mmu/mali_kbase_mmu.c26
-rw-r--r--mali_kbase/tests/Mconfig8
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/Makefile4
-rw-r--r--mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c17
-rw-r--r--mali_kbase/tl/mali_kbase_timeline.c12
-rw-r--r--mali_kbase/tl/mali_kbase_timeline.h14
-rw-r--r--mali_kbase/tl/mali_kbase_timeline_io.c2
-rw-r--r--mali_kbase/tl/mali_kbase_timeline_priv.h4
-rw-r--r--mali_kbase/tl/mali_kbase_tl_serialize.h6
-rw-r--r--mali_kbase/tl/mali_kbase_trace_defs.h261
-rw-r--r--mali_kbase/tl/mali_kbase_tracepoints.c176
-rw-r--r--mali_kbase/tl/mali_kbase_tracepoints.h1288
87 files changed, 6009 insertions, 2636 deletions
diff --git a/mali_kbase/Kbuild b/mali_kbase/Kbuild
index 7abe8d3..192ac06 100644
--- a/mali_kbase/Kbuild
+++ b/mali_kbase/Kbuild
@@ -21,7 +21,7 @@
# Driver version string which is returned to userspace via an ioctl
-MALI_RELEASE_NAME ?= "r24p0-01rel0"
+MALI_RELEASE_NAME ?= "r25p0-01eac0"
# Paths required for build
KBASE_PATH = $(src)
@@ -73,6 +73,7 @@ subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UM
SRC := \
context/mali_kbase_context.c \
+ debug/mali_kbase_debug_ktrace.c \
device/mali_kbase_device.c \
mali_kbase_cache_policy.c \
mali_kbase_mem.c \
@@ -81,7 +82,6 @@ SRC := \
mali_kbase_ctx_sched.c \
mali_kbase_jm.c \
mali_kbase_gpuprops.c \
- mali_kbase_js.c \
mali_kbase_pm.c \
mali_kbase_config.c \
mali_kbase_vinstr.c \
@@ -120,6 +120,7 @@ SRC := \
ifeq ($(MALI_USE_CSF),1)
SRC += \
+ debug/backend/mali_kbase_debug_ktrace_csf.c \
device/backend/mali_kbase_device_csf.c \
gpu/backend/mali_kbase_gpu_fault_csf.c \
tl/backend/mali_kbase_timeline_csf.c \
@@ -132,7 +133,9 @@ else
mali_kbase_event.c \
mali_kbase_jd.c \
mali_kbase_jd_debugfs.c \
+ mali_kbase_js.c \
mali_kbase_js_ctx_attr.c \
+ debug/backend/mali_kbase_debug_ktrace_jm.c \
device/backend/mali_kbase_device_jm.c \
gpu/backend/mali_kbase_gpu_fault_jm.c \
tl/backend/mali_kbase_timeline_jm.c \
@@ -153,7 +156,8 @@ ifeq ($(MALI_CUSTOMER_RELEASE),0)
endif
-ccflags-y += -I$(KBASE_PATH)
+ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \
+ -I$(KBASE_PATH)/debug/backend
# Tell the Linux build system from which .o file to create the kernel module
obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o
@@ -179,6 +183,12 @@ else
# empty
endif
+ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y)
+ include $(src)/arbiter/Kbuild
+else
+# empty
+endif
+
mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \
mali_kbase_dma_fence.o \
mali_kbase_fence.o
diff --git a/mali_kbase/Kconfig b/mali_kbase/Kconfig
index a46305d..58a5b0b 100644
--- a/mali_kbase/Kconfig
+++ b/mali_kbase/Kconfig
@@ -49,6 +49,7 @@ config MALI_MIDGARD_DVFS
config MALI_MIDGARD_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_MIDGARD
+ default y if MALI_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
@@ -84,6 +85,17 @@ config MALI_PLATFORM_NAME
include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
exist.
+config MALI_ARBITER_SUPPORT
+ bool "Enable arbiter support for Mali"
+ depends on MALI_MIDGARD
+ default n
+ help
+ Enable support for the arbiter interface in the driver.
+ This allows an external arbiter to manage driver access
+ to GPU hardware in a virtualized environment
+
+ If unsure, say N.
+
# MALI_EXPERT configuration options
menuconfig MALI_EXPERT
@@ -156,6 +168,7 @@ config MALI_ERROR_INJECT
config MALI_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_MIDGARD && MALI_EXPERT
+ default y if MALI_DEBUG
default n
help
Choose this option to enable system trace events for each
@@ -240,6 +253,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
+config MALI_GEM5_BUILD
+ bool "Enable build of Mali kernel driver for GEM5"
+ depends on MALI_MIDGARD
+ default n
+ help
+ This option is to do a Mali GEM5 build.
+ If unsure, say N.
+
# Instrumentation options.
config MALI_JOB_DUMP
diff --git a/mali_kbase/Mconfig b/mali_kbase/Mconfig
index 27399fa..b137793 100644
--- a/mali_kbase/Mconfig
+++ b/mali_kbase/Mconfig
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2012-2019 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -40,6 +40,7 @@ config MALI_MIDGARD_DVFS
config MALI_MIDGARD_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_MIDGARD
+ default y if MALI_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
@@ -81,6 +82,17 @@ config MALI_PLATFORM_NAME
When PLATFORM_CUSTOM is set, this needs to be set manually to
pick up the desired platform files.
+config MALI_ARBITER_SUPPORT
+ bool "Enable arbiter support for Mali"
+ depends on MALI_MIDGARD
+ default n
+ help
+ Enable support for the arbiter interface in the driver.
+ This allows an external arbiter to manage driver access
+ to GPU hardware in a virtualized environment
+
+ If unsure, say N.
+
# MALI_EXPERT configuration options
menuconfig MALI_EXPERT
@@ -167,10 +179,11 @@ config MALI_ERROR_INJECT
config MALI_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_MIDGARD && MALI_EXPERT
+ default y if MALI_DEBUG
default n
help
Choose this option to enable system trace events for each
- kbase event. This is typically used for debugging but has
+ kbase event. This is typically used for debugging but has
minimal overhead when not in use. Enable only if you know what
you are doing.
@@ -249,6 +262,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
+config MALI_GEM5_BUILD
+ bool "Enable build of Mali kernel driver for GEM5"
+ depends on MALI_MIDGARD
+ default n
+ help
+ This option is to do a Mali GEM5 build.
+ If unsure, say N.
+
# Instrumentation options.
# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig.
diff --git a/mali_kbase/arbiter/Kbuild b/mali_kbase/arbiter/Kbuild
new file mode 100644
index 0000000..98e47be
--- /dev/null
+++ b/mali_kbase/arbiter/Kbuild
@@ -0,0 +1,24 @@
+#
+# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, you can access it online at
+# http://www.gnu.org/licenses/gpl-2.0.html.
+#
+# SPDX-License-Identifier: GPL-2.0
+#
+#
+
+mali_kbase-y += \
+ arbiter/mali_kbase_arbif.o \
+ arbiter/mali_kbase_arbiter_pm.o
diff --git a/mali_kbase/arbiter/mali_kbase_arbif.c b/mali_kbase/arbiter/mali_kbase_arbif.c
new file mode 100644
index 0000000..d193cb9
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbif.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/**
+ * @file mali_kbase_arbif.c
+ * Mali arbiter interface APIs to share GPU between Virtual Machines
+ */
+
+#include <mali_kbase.h>
+#include "mali_kbase_arbif.h"
+#include <tl/mali_kbase_tracepoints.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include "mali_kbase_arbiter_interface.h"
+
+static void on_gpu_stop(struct device *dev)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED(kbdev, kbdev);
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT);
+}
+
+static void on_gpu_granted(struct device *dev)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED(kbdev, kbdev);
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT);
+}
+
+static void on_gpu_lost(struct device *dev)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT);
+}
+
+int kbase_arbif_init(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_OF
+ struct arbiter_if_arb_vm_ops ops;
+ struct arbiter_if_dev *arb_if;
+ struct device_node *arbiter_if_node;
+ struct platform_device *pdev;
+ int err;
+
+ dev_dbg(kbdev->dev, "%s\n", __func__);
+
+ arbiter_if_node = of_parse_phandle(kbdev->dev->of_node,
+ "arbiter_if", 0);
+ if (!arbiter_if_node) {
+ dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n");
+ /* no arbiter interface defined in device tree */
+ kbdev->arb.arb_dev = NULL;
+ kbdev->arb.arb_if = NULL;
+ return 0;
+ }
+
+ pdev = of_find_device_by_node(arbiter_if_node);
+ if (!pdev) {
+ dev_err(kbdev->dev, "Failed to find arbiter_if device\n");
+ return -EPROBE_DEFER;
+ }
+
+ if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) {
+ dev_err(kbdev->dev, "arbiter_if driver not available\n");
+ return -EPROBE_DEFER;
+ }
+ kbdev->arb.arb_dev = &pdev->dev;
+ arb_if = platform_get_drvdata(pdev);
+ if (!arb_if) {
+ dev_err(kbdev->dev, "arbiter_if driver not ready\n");
+ module_put(pdev->dev.driver->owner);
+ return -EPROBE_DEFER;
+ }
+
+ kbdev->arb.arb_if = arb_if;
+ ops.arb_vm_gpu_stop = on_gpu_stop;
+ ops.arb_vm_gpu_granted = on_gpu_granted;
+ ops.arb_vm_gpu_lost = on_gpu_lost;
+
+ /* register kbase arbiter_if callbacks */
+ if (arb_if->vm_ops.vm_arb_register_dev) {
+ err = arb_if->vm_ops.vm_arb_register_dev(arb_if,
+ kbdev->dev, &ops);
+ if (err) {
+ dev_err(kbdev->dev, "Arbiter registration failed.\n");
+ module_put(pdev->dev.driver->owner);
+ return err;
+ }
+ }
+#else /* CONFIG_OF */
+ dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n");
+ kbdev->arb.arb_dev = NULL;
+ kbdev->arb.arb_if = NULL;
+#endif
+ return 0;
+}
+
+void kbase_arbif_destroy(struct kbase_device *kbdev)
+{
+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
+
+ if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) {
+ dev_dbg(kbdev->dev, "%s\n", __func__);
+ arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if);
+ }
+ kbdev->arb.arb_if = NULL;
+ if (kbdev->arb.arb_dev)
+ module_put(kbdev->arb.arb_dev->driver->owner);
+ kbdev->arb.arb_dev = NULL;
+}
+
+void kbase_arbif_gpu_request(struct kbase_device *kbdev)
+{
+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
+
+ if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) {
+ dev_dbg(kbdev->dev, "%s\n", __func__);
+ arb_if->vm_ops.vm_arb_gpu_request(arb_if);
+ }
+}
+
+void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required)
+{
+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
+
+ if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) {
+ dev_dbg(kbdev->dev, "%s\n", __func__);
+ KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED(kbdev, kbdev);
+ arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required);
+ }
+}
+
+void kbase_arbif_gpu_active(struct kbase_device *kbdev)
+{
+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
+
+ if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) {
+ dev_dbg(kbdev->dev, "%s\n", __func__);
+ arb_if->vm_ops.vm_arb_gpu_active(arb_if);
+ }
+}
+
+void kbase_arbif_gpu_idle(struct kbase_device *kbdev)
+{
+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
+
+ if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) {
+ dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n");
+ arb_if->vm_ops.vm_arb_gpu_idle(arb_if);
+ }
+}
diff --git a/mali_kbase/arbiter/mali_kbase_arbif.h b/mali_kbase/arbiter/mali_kbase_arbif.h
new file mode 100644
index 0000000..e7e9de7
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbif.h
@@ -0,0 +1,133 @@
+/*
+ *
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ *//* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ *
+ */
+
+/**
+ * @file
+ * Mali arbiter interface APIs to share GPU between Virtual Machines
+ */
+
+#ifndef _MALI_KBASE_ARBIF_H_
+#define _MALI_KBASE_ARBIF_H_
+
+/**
+ * enum kbase_arbif_evt - Internal Arbiter event.
+ *
+ * @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing
+ * and can be stopped
+ * @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter
+ * @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter
+ * @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter
+ * @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state.
+ * @KBASE_VM_REF_EVENT: KBase has transitioned into an active state.
+ * @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending
+ * @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming
+ */
+enum kbase_arbif_evt {
+ KBASE_VM_GPU_INITIALIZED_EVT = 1,
+ KBASE_VM_GPU_STOP_EVT,
+ KBASE_VM_GPU_GRANTED_EVT,
+ KBASE_VM_GPU_LOST_EVT,
+ KBASE_VM_GPU_IDLE_EVENT,
+ KBASE_VM_REF_EVENT,
+ KBASE_VM_OS_SUSPEND_EVENT,
+ KBASE_VM_OS_RESUME_EVENT,
+};
+
+/**
+ * kbase_arbif_init() - Initialize the arbiter interface functionality.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Initialize the arbiter interface and also determines
+ * if Arbiter functionality is required.
+ *
+ * Return: 0 if the Arbiter interface was successfully initialized or the
+ * Arbiter was not required.
+ */
+int kbase_arbif_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbif_destroy() - Cleanups the arbiter interface functionality.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Cleans up the arbiter interface functionality and resets the reference count
+ * of the arbif module used
+ */
+void kbase_arbif_destroy(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbif_gpu_request() - Send GPU request message to the arbiter
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Sends a message to Arbiter to request GPU access.
+ */
+void kbase_arbif_gpu_request(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ * @gpu_required: true if GPU access is still required
+ * (Arbiter will automatically send another grant message)
+ *
+ * Sends a message to Arbiter to notify that the GPU has stopped.
+ * @note Once this call has been made, KBase must not attempt to access the GPU
+ * until the #KBASE_VM_GPU_GRANTED_EVT event has been received.
+ */
+void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required);
+
+/**
+ * kbase_arbif_gpu_active() - Send a GPU active message to the arbiter
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Sends a message to Arbiter to report that KBase has gone active.
+ */
+void kbase_arbif_gpu_active(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Sends a message to Arbiter to report that KBase has gone idle.
+ */
+void kbase_arbif_gpu_idle(struct kbase_device *kbdev);
+
+#endif /* _MALI_KBASE_ARBIF_H_ */
diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_defs.h b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h
new file mode 100644
index 0000000..1f53cbf
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbiter_defs.h
@@ -0,0 +1,95 @@
+/*
+ *
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ *//* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ *
+ */
+
+/**
+ * @file
+ * Mali structures define to support arbitration feature
+ */
+
+#ifndef _MALI_KBASE_ARBITER_DEFS_H_
+#define _MALI_KBASE_ARBITER_DEFS_H_
+
+#include "mali_kbase_arbiter_pm.h"
+
+/**
+ * struct kbase_arbiter_vm_state - Struct representing the state and containing the
+ * data of pm work
+ * @kbdev: Pointer to kbase device structure (must be a valid pointer)
+ * @vm_state_lock: The lock protecting the VM state when arbiter is used.
+ * This lock must also be held whenever the VM state is being
+ * transitioned
+ * @vm_state_wait: Wait queue set when GPU is granted
+ * @vm_state: Current state of VM
+ * @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use
+ * with the Arbiter
+ * @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU
+ * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU
+ * @vm_arb_starting: Work queue resume in progress
+ * @vm_arb_stopping: Work queue suspend in progress
+ * @vm_arb_users_waiting: Count of users waiting for GPU
+ */
+struct kbase_arbiter_vm_state {
+ struct kbase_device *kbdev;
+ struct mutex vm_state_lock;
+ wait_queue_head_t vm_state_wait;
+ enum kbase_vm_state vm_state;
+ struct workqueue_struct *vm_arb_wq;
+ struct work_struct vm_suspend_work;
+ struct work_struct vm_resume_work;
+ bool vm_arb_starting;
+ bool vm_arb_stopping;
+ int vm_arb_users_waiting;
+};
+
+/**
+ * struct kbase_arbiter_device - Representing an instance of arbiter device,
+ * allocated from the probe method of Mali driver
+ * @arb_if: Pointer to the arbiter interface device
+ * @arb_dev: Pointer to the arbiter device
+ */
+struct kbase_arbiter_device {
+ struct arbiter_if_dev *arb_if;
+ struct device *arb_dev;
+};
+
+#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */
diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_interface.h b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h
new file mode 100644
index 0000000..5d5d8a7
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbiter_interface.h
@@ -0,0 +1,181 @@
+/*
+ *
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ *//* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ *
+ */
+
+/**
+ * @file
+ * Defines the Mali arbiter interface
+ */
+
+#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_
+#define _MALI_KBASE_ARBITER_INTERFACE_H_
+
+/**
+ * @brief Mali arbiter interface version
+ *
+ * This specifies the current version of the configuration interface. Whenever
+ * the arbiter interface changes, so that integration effort is required, the
+ * version number will be increased. Each configuration must make an effort
+ * to check that it implements the correct version.
+ *
+ * Version history:
+ * 1 - Added the Mali arbiter configuration interface.
+ * 2 - Strip out reference code from header
+ * 3 - Removed DVFS utilization interface (DVFS moved to arbiter side)
+ */
+#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3
+
+struct arbiter_if_dev;
+
+/**
+ * struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM
+ *
+ * This struct contains callbacks used to deliver messages
+ * from the arbiter to the corresponding VM.
+ *
+ * Note that calls into these callbacks may have synchronous calls back into
+ * the arbiter arbiter_if_vm_arb_ops callbacks below.
+ * For example vm_arb_gpu_stopped() may be called as a side effect of
+ * arb_vm_gpu_stop() being called here.
+ */
+struct arbiter_if_arb_vm_ops {
+ /**
+ * arb_vm_gpu_stop() - Ask VM to stop using GPU
+ * @dev: The arbif kernel module device.
+ *
+ * Informs KBase to stop using the GPU as soon as possible.
+ * @Note: Once the driver is no longer using the GPU, a call to
+ * vm_arb_gpu_stopped is expected by the arbiter.
+ */
+ void (*arb_vm_gpu_stop)(struct device *dev);
+
+ /**
+ * arb_vm_gpu_granted() - GPU has been granted to VM
+ * @dev: The arbif kernel module device.
+ *
+ * Informs KBase that the GPU can now be used by the VM.
+ */
+ void (*arb_vm_gpu_granted)(struct device *dev);
+
+ /**
+ * arb_vm_gpu_lost() - VM has lost the GPU
+ * @dev: The arbif kernel module device.
+ *
+ * This is called if KBase takes too long to respond to the arbiter
+ * stop request.
+ * Once this is called, KBase will assume that access to the GPU
+ * has been lost and will fail all running jobs and reset its
+ * internal state.
+ * If successful, will respond with a vm_arb_gpu_stopped message.
+ */
+ void (*arb_vm_gpu_lost)(struct device *dev);
+};
+
+/**
+ * struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter
+ *
+ * This struct contains callbacks used to request operations
+ * from the VM to the arbiter
+ *
+ * Note that we must not make any synchronous calls back in to the VM
+ * (via arbiter_if_arb_vm_ops above) in the context of these callbacks.
+ */
+struct arbiter_if_vm_arb_ops {
+ /**
+ * vm_arb_register_dev() - Register VM device driver callbacks.
+ * @arbif_dev: The arbiter interface we are registering device callbacks
+ * @dev: The device structure to supply in the callbacks.
+ * @ops: The callbacks that the device driver supports
+ * (none are optional).
+ */
+ int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev,
+ struct device *dev, struct arbiter_if_arb_vm_ops *ops);
+
+ /**
+ * vm_arb_unregister_dev() - Unregister VM device driver callbacks.
+ * @arbif_dev: The arbiter interface we are unregistering from.
+ */
+ void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev);
+
+ /**
+ * vm_arb_gpu_request() - Ask the arbiter interface for GPU access.
+ * @arbif_dev: The arbiter interface we want to issue the request.
+ */
+ void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev);
+
+ /**
+ * vm_arb_gpu_active() - Inform arbiter that the driver has gone active
+ * @arbif_dev: The arbiter interface device.
+ */
+ void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev);
+
+ /**
+ * vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle
+ * @arbif_dev: The arbiter interface device.
+ */
+ void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev);
+
+ /**
+ * vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped
+ * using the GPU
+ * @arbif_dev: The arbiter interface device.
+ * @gpu_required: The GPU is still needed to do more work.
+ */
+ void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev,
+ u8 gpu_required);
+};
+
+/**
+ * struct arbiter_if_dev - Arbiter Interface
+ * @vm_ops: Callback functions for connecting KBase with
+ * arbiter interface device.
+ * @priv_data: Internal arbif data not used by KBASE.
+ *
+ * Arbiter Interface Kernel Module State used for linking KBase
+ * with an arbiter interface platform device
+ */
+struct arbiter_if_dev {
+ struct arbiter_if_vm_arb_ops vm_ops;
+ void *priv_data;
+};
+
+#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */
diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.c b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c
new file mode 100644
index 0000000..6c35e16
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.c
@@ -0,0 +1,645 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/**
+ * @file mali_kbase_arbiter_pm.c
+ * Mali arbiter power manager state machine and APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_irq_internal.h>
+#include <mali_kbase_hwcnt_context.h>
+#include <mali_kbase_pm_internal.h>
+#include <tl/mali_kbase_tracepoints.h>
+
+static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev);
+static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
+ struct kbase_device *kbdev);
+
+static inline const char *kbase_arbiter_pm_vm_state_str(
+ enum kbase_vm_state state)
+{
+ switch (state) {
+ case KBASE_VM_STATE_INITIALIZING:
+ return "KBASE_VM_STATE_INITIALIZING";
+ case KBASE_VM_STATE_INITIALIZING_WITH_GPU:
+ return "KBASE_VM_STATE_INITIALIZING_WITH_GPU";
+ case KBASE_VM_STATE_SUSPENDED:
+ return "KBASE_VM_STATE_SUSPENDED";
+ case KBASE_VM_STATE_STOPPED:
+ return "KBASE_VM_STATE_STOPPED";
+ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
+ return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED";
+ case KBASE_VM_STATE_STARTING:
+ return "KBASE_VM_STATE_STARTING";
+ case KBASE_VM_STATE_IDLE:
+ return "KBASE_VM_STATE_IDLE";
+ case KBASE_VM_STATE_ACTIVE:
+ return "KBASE_VM_STATE_ACTIVE";
+ case KBASE_VM_STATE_STOPPING_IDLE:
+ return "KBASE_VM_STATE_STOPPING_IDLE";
+ case KBASE_VM_STATE_STOPPING_ACTIVE:
+ return "KBASE_VM_STATE_STOPPING_ACTIVE";
+ case KBASE_VM_STATE_SUSPEND_PENDING:
+ return "KBASE_VM_STATE_SUSPEND_PENDING";
+ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
+ return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT";
+ default:
+ KBASE_DEBUG_ASSERT(false);
+ return "[UnknownState]";
+ }
+}
+
+static inline const char *kbase_arbiter_pm_vm_event_str(
+ enum kbase_arbif_evt evt)
+{
+ switch (evt) {
+ case KBASE_VM_GPU_INITIALIZED_EVT:
+ return "KBASE_VM_GPU_INITIALIZED_EVT";
+ case KBASE_VM_GPU_STOP_EVT:
+ return "KBASE_VM_GPU_STOP_EVT";
+ case KBASE_VM_GPU_GRANTED_EVT:
+ return "KBASE_VM_GPU_GRANTED_EVT";
+ case KBASE_VM_GPU_LOST_EVT:
+ return "KBASE_VM_GPU_LOST_EVT";
+ case KBASE_VM_OS_SUSPEND_EVENT:
+ return "KBASE_VM_OS_SUSPEND_EVENT";
+ case KBASE_VM_OS_RESUME_EVENT:
+ return "KBASE_VM_OS_RESUME_EVENT";
+ case KBASE_VM_GPU_IDLE_EVENT:
+ return "KBASE_VM_GPU_IDLE_EVENT";
+ case KBASE_VM_REF_EVENT:
+ return "KBASE_VM_REF_EVENT";
+ default:
+ KBASE_DEBUG_ASSERT(false);
+ return "[UnknownEvent]";
+ }
+}
+
+static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev,
+ enum kbase_vm_state new_state)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ dev_dbg(kbdev->dev, "VM set_state %s -> %s",
+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state),
+ kbase_arbiter_pm_vm_state_str(new_state));
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ arb_vm_state->vm_state = new_state;
+ wake_up(&arb_vm_state->vm_state_wait);
+}
+
+static void kbase_arbiter_pm_suspend_wq(struct work_struct *data)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
+ struct kbase_arbiter_vm_state,
+ vm_suspend_work);
+ struct kbase_device *kbdev = arb_vm_state->kbdev;
+
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, ">%s\n", __func__);
+ if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE ||
+ arb_vm_state->vm_state ==
+ KBASE_VM_STATE_STOPPING_ACTIVE ||
+ arb_vm_state->vm_state ==
+ KBASE_VM_STATE_SUSPEND_PENDING) {
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n");
+ kbase_pm_driver_suspend(kbdev);
+ dev_dbg(kbdev->dev, "<kbase_pm_driver_suspend\n");
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ }
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, "<%s\n", __func__);
+}
+
+static void kbase_arbiter_pm_resume_wq(struct work_struct *data)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
+ struct kbase_arbiter_vm_state,
+ vm_resume_work);
+ struct kbase_device *kbdev = arb_vm_state->kbdev;
+
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, ">%s\n", __func__);
+ arb_vm_state->vm_arb_starting = true;
+ if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) {
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n");
+ kbase_pm_driver_resume(kbdev, true);
+ dev_dbg(kbdev->dev, "<kbase_pm_driver_resume\n");
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) {
+ kbase_arbiter_pm_vm_stopped(kbdev);
+ }
+ arb_vm_state->vm_arb_starting = false;
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, "<%s\n", __func__);
+}
+
+int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
+{
+ int err;
+ struct kbase_arbiter_vm_state *arb_vm_state = NULL;
+
+ arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state),
+ GFP_KERNEL);
+ if (arb_vm_state == NULL)
+ return -ENOMEM;
+
+ arb_vm_state->kbdev = kbdev;
+ arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING;
+
+ mutex_init(&arb_vm_state->vm_state_lock);
+ init_waitqueue_head(&arb_vm_state->vm_state_wait);
+ arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq",
+ WQ_HIGHPRI);
+ if (!arb_vm_state->vm_arb_wq) {
+ dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n");
+ return -ENOMEM;
+ }
+ INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq);
+ INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq);
+ arb_vm_state->vm_arb_starting = false;
+ arb_vm_state->vm_arb_users_waiting = 0;
+ kbdev->pm.arb_vm_state = arb_vm_state;
+
+ err = kbase_arbif_init(kbdev);
+ if (err) {
+ goto arbif_init_fail;
+ }
+ if (kbdev->arb.arb_if) {
+ kbase_arbif_gpu_request(kbdev);
+ dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n");
+ wait_event(arb_vm_state->vm_state_wait,
+ arb_vm_state->vm_state ==
+ KBASE_VM_STATE_INITIALIZING_WITH_GPU);
+ dev_dbg(kbdev->dev,
+ "Waiting for initial GPU assignment - done\n");
+ }
+ return 0;
+
+arbif_init_fail:
+ destroy_workqueue(arb_vm_state->vm_arb_wq);
+ kfree(arb_vm_state);
+ kbdev->pm.arb_vm_state = NULL;
+ return err;
+}
+
+void kbase_arbiter_pm_early_term(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
+ kbase_arbif_gpu_stopped(kbdev, false);
+
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ kbase_arbif_destroy(kbdev);
+ destroy_workqueue(arb_vm_state->vm_arb_wq);
+ arb_vm_state->vm_arb_wq = NULL;
+ kfree(kbdev->pm.arb_vm_state);
+ kbdev->pm.arb_vm_state = NULL;
+}
+
+void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ if (!kbdev->arb.arb_if ||
+ arb_vm_state->vm_state >
+ KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
+ kbase_release_interrupts(kbdev);
+
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+}
+
+void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev)
+{
+ bool request_gpu = false;
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+
+ if (arb_vm_state->vm_arb_users_waiting > 0 &&
+ arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE)
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_ACTIVE);
+
+ dev_dbg(kbdev->dev, "%s %s\n", __func__,
+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
+ kbase_release_interrupts(kbdev);
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_STOPPING_ACTIVE:
+ request_gpu = true;
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
+ break;
+ case KBASE_VM_STATE_STOPPING_IDLE:
+ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED);
+ break;
+ case KBASE_VM_STATE_SUSPEND_PENDING:
+ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
+ break;
+ default:
+ dev_warn(kbdev->dev, "unexpected pm_stop VM state %u",
+ arb_vm_state->vm_state);
+ break;
+ }
+
+ kbase_arbif_gpu_stopped(kbdev, request_gpu);
+}
+
+static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_INITIALIZING:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_INITIALIZING_WITH_GPU);
+ break;
+ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
+ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING);
+ kbase_install_interrupts(kbdev);
+ queue_work(arb_vm_state->vm_arb_wq,
+ &arb_vm_state->vm_resume_work);
+ break;
+ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
+ kbase_arbif_gpu_stopped(kbdev, false);
+ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
+ break;
+ default:
+ dev_warn(kbdev->dev,
+ "GPU_GRANTED when not expected - state %s\n",
+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
+ break;
+ }
+}
+
+static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) {
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ }
+
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_IDLE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_IDLE);
+ queue_work(arb_vm_state->vm_arb_wq,
+ &arb_vm_state->vm_suspend_work);
+ break;
+ case KBASE_VM_STATE_ACTIVE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_ACTIVE);
+ queue_work(arb_vm_state->vm_arb_wq,
+ &arb_vm_state->vm_suspend_work);
+ break;
+ case KBASE_VM_STATE_STARTING:
+ dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING.");
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_ACTIVE);
+ if (arb_vm_state->vm_arb_starting)
+ queue_work(arb_vm_state->vm_arb_wq,
+ &arb_vm_state->vm_suspend_work);
+ break;
+ case KBASE_VM_STATE_SUSPEND_PENDING:
+ /* Suspend finishes with a stop so nothing else to do */
+ break;
+ default:
+ dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n",
+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
+ break;
+ }
+}
+
+static void kbase_gpu_lost(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_STARTING:
+ case KBASE_VM_STATE_ACTIVE:
+ case KBASE_VM_STATE_IDLE:
+ dev_warn(kbdev->dev, "GPU lost in state %s",
+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
+ kbase_arbiter_pm_vm_gpu_stop(kbdev);
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ kbase_pm_handle_gpu_lost(kbdev);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ break;
+ case KBASE_VM_STATE_STOPPING_IDLE:
+ case KBASE_VM_STATE_STOPPING_ACTIVE:
+ case KBASE_VM_STATE_SUSPEND_PENDING:
+ dev_info(kbdev->dev, "GPU lost while stopping");
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ kbase_pm_handle_gpu_lost(kbdev);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ break;
+ case KBASE_VM_STATE_SUSPENDED:
+ case KBASE_VM_STATE_STOPPED:
+ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
+ dev_info(kbdev->dev, "GPU lost while already stopped");
+ break;
+ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
+ dev_info(kbdev->dev, "GPU lost while waiting to suspend");
+ kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
+ break;
+ default:
+ break;
+ }
+
+}
+
+static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(
+ struct kbase_device *kbdev)
+{
+ switch (kbdev->pm.arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_SUSPENDED:
+ case KBASE_VM_STATE_STOPPED:
+ case KBASE_VM_STATE_IDLE:
+ case KBASE_VM_STATE_ACTIVE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+ enum kbase_vm_state prev_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ if (kbdev->arb.arb_if) {
+ if (kbdev->pm.arb_vm_state->vm_state ==
+ KBASE_VM_STATE_SUSPENDED)
+ return;
+ }
+ /* Block suspend OS function until we are in a stable state
+ * with vm_state_lock
+ */
+ while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) {
+ prev_state = arb_vm_state->vm_state;
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_STOPPING_ACTIVE:
+ case KBASE_VM_STATE_STOPPING_IDLE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_SUSPEND_PENDING);
+ break;
+ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT);
+ break;
+ case KBASE_VM_STATE_STARTING:
+ if (!arb_vm_state->vm_arb_starting) {
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_SUSPEND_PENDING);
+ kbase_arbiter_pm_vm_stopped(kbdev);
+ }
+ break;
+ default:
+ break;
+ }
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ wait_event(arb_vm_state->vm_state_wait,
+ arb_vm_state->vm_state != prev_state);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ }
+
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_STOPPED:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_SUSPENDED);
+ break;
+ case KBASE_VM_STATE_IDLE:
+ case KBASE_VM_STATE_ACTIVE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_SUSPEND_PENDING);
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ /* Ensure resume has completed fully before starting suspend */
+ flush_work(&arb_vm_state->vm_resume_work);
+ kbase_pm_driver_suspend(kbdev);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ break;
+ case KBASE_VM_STATE_SUSPENDED:
+ break;
+ default:
+ KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend");
+ break;
+ }
+}
+
+static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state ==
+ KBASE_VM_STATE_SUSPENDED,
+ "Unexpected state to resume");
+
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
+ kbase_arbif_gpu_request(kbdev);
+
+ /* Release lock and block resume OS function until we have
+ * asynchronously received the GRANT message from the Arbiter and
+ * fully resumed
+ */
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
+ flush_work(&arb_vm_state->vm_resume_work);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+}
+
+void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
+ enum kbase_arbif_evt evt)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ if (!kbdev->arb.arb_if)
+ return;
+
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ dev_dbg(kbdev->dev, "%s %s\n", __func__,
+ kbase_arbiter_pm_vm_event_str(evt));
+
+ switch (evt) {
+ case KBASE_VM_GPU_GRANTED_EVT:
+ kbase_arbiter_pm_vm_gpu_start(kbdev);
+ break;
+ case KBASE_VM_GPU_STOP_EVT:
+ kbase_arbiter_pm_vm_gpu_stop(kbdev);
+ break;
+ case KBASE_VM_GPU_LOST_EVT:
+ dev_info(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!");
+ kbase_gpu_lost(kbdev);
+ break;
+ case KBASE_VM_OS_SUSPEND_EVENT:
+ kbase_arbiter_pm_vm_os_prepare_suspend(kbdev);
+ break;
+ case KBASE_VM_OS_RESUME_EVENT:
+ kbase_arbiter_pm_vm_os_resume(kbdev);
+ break;
+ case KBASE_VM_GPU_IDLE_EVENT:
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_ACTIVE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_IDLE);
+ kbase_arbif_gpu_idle(kbdev);
+ break;
+ default:
+ break;
+ }
+ break;
+
+ case KBASE_VM_REF_EVENT:
+ switch (arb_vm_state->vm_state) {
+ case KBASE_VM_STATE_STARTING:
+ KBASE_TLSTREAM_TL_EVENT_ARB_STARTED(kbdev, kbdev);
+ /* FALL THROUGH */
+ case KBASE_VM_STATE_IDLE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_ACTIVE);
+ kbase_arbif_gpu_active(kbdev);
+ break;
+ case KBASE_VM_STATE_STOPPING_IDLE:
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_ACTIVE);
+ break;
+ default:
+ break;
+ }
+ break;
+
+ case KBASE_VM_GPU_INITIALIZED_EVT:
+ lockdep_assert_held(&kbdev->pm.lock);
+ if (kbdev->pm.active_count > 0) {
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_ACTIVE);
+ kbase_arbif_gpu_active(kbdev);
+ } else {
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_IDLE);
+ kbase_arbif_gpu_idle(kbdev);
+ }
+ break;
+
+ default:
+ dev_alert(kbdev->dev, "Got Unknown Event!");
+ break;
+ }
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+}
+
+static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n");
+ wait_event(arb_vm_state->vm_state_wait,
+ arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
+ arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
+ dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n");
+}
+
+static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
+ struct kbase_device *kbdev)
+{
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ lockdep_assert_held(&arb_vm_state->vm_state_lock);
+ return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
+ arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
+}
+
+int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
+ enum kbase_pm_suspend_handler suspend_handler)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
+
+ if (kbdev->arb.arb_if) {
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) {
+ /* Update VM state since we have GPU work to do */
+ if (arb_vm_state->vm_state ==
+ KBASE_VM_STATE_STOPPING_IDLE)
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPING_ACTIVE);
+ else if (arb_vm_state->vm_state ==
+ KBASE_VM_STATE_STOPPED) {
+ kbase_arbiter_pm_vm_set_state(kbdev,
+ KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
+ kbase_arbif_gpu_request(kbdev);
+ } else if (arb_vm_state->vm_state ==
+ KBASE_VM_STATE_INITIALIZING_WITH_GPU)
+ break;
+
+ if (suspend_handler !=
+ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) {
+ if (suspend_handler ==
+ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED
+ ||
+ kbdev->pm.active_count > 0)
+ break;
+
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ return 1;
+ }
+
+ if (arb_vm_state->vm_state ==
+ KBASE_VM_STATE_INITIALIZING_WITH_GPU)
+ break;
+
+ /* Need to synchronously wait for GPU assignment */
+ arb_vm_state->vm_arb_users_waiting++;
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
+ mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+ mutex_lock(&arb_vm_state->vm_state_lock);
+ arb_vm_state->vm_arb_users_waiting--;
+ }
+ mutex_unlock(&arb_vm_state->vm_state_lock);
+ }
+ return 0;
+}
diff --git a/mali_kbase/arbiter/mali_kbase_arbiter_pm.h b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h
new file mode 100644
index 0000000..3c49eb1
--- /dev/null
+++ b/mali_kbase/arbiter/mali_kbase_arbiter_pm.h
@@ -0,0 +1,159 @@
+/*
+ *
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ *//* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ *
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU license.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/**
+ * @file
+ * Mali arbiter power manager state machine and APIs
+ */
+
+#ifndef _MALI_KBASE_ARBITER_PM_H_
+#define _MALI_KBASE_ARBITER_PM_H_
+
+#include "mali_kbase_arbif.h"
+
+/**
+ * enum kbase_vm_state - Current PM Arbitration state.
+ *
+ * @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized.
+ * @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU
+ * has been granted.
+ * @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned.
+ * @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required.
+ * @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase
+ * but a request has been made.
+ * @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run.
+ * @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do
+ * @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it
+ * @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode.
+ * @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU
+ * has already been requested.
+ * In this situation we must wait for
+ * the Arbiter to send a GRANTED message
+ * and respond immediately with
+ * a STOPPED message before entering
+ * the suspend mode.
+ * @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there
+ * is currently no work to do on the GPU.
+ * @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when
+ * KBase has work to do.
+ */
+enum kbase_vm_state {
+ KBASE_VM_STATE_INITIALIZING,
+ KBASE_VM_STATE_INITIALIZING_WITH_GPU,
+ KBASE_VM_STATE_SUSPENDED,
+ KBASE_VM_STATE_STOPPED,
+ KBASE_VM_STATE_STOPPED_GPU_REQUESTED,
+ KBASE_VM_STATE_STARTING,
+ KBASE_VM_STATE_IDLE,
+ KBASE_VM_STATE_ACTIVE,
+ KBASE_VM_STATE_SUSPEND_PENDING,
+ KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT,
+ KBASE_VM_STATE_STOPPING_IDLE,
+ KBASE_VM_STATE_STOPPING_ACTIVE
+};
+
+/**
+ * kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Initialize the arbiter and other required resources during the runtime
+ * and request the GPU for the VM for the first time.
+ *
+ * Return: 0 if successful, otherwise a standard Linux error code
+ */
+int kbase_arbiter_pm_early_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Clean up all the resources
+ */
+void kbase_arbiter_pm_early_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Releases interrupts if needed (GPU is available) otherwise does nothing
+ */
+void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev);
+
+/**
+ * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * The state machine function. Receives events and transitions states
+ * according the event received and the current state
+ */
+void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
+ enum kbase_arbif_evt event);
+
+/**
+ * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for
+ * arbitration mode
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ * @suspend_handler: The handler code for how to handle a suspend
+ * that might occur
+ *
+ * This function handles a suspend event from the driver,
+ * communicating with the arbiter and waiting synchronously for the GPU
+ * to be granted again depending on the VM state.
+ *
+ * Return: 0 if success, 1 if failure due to system suspending/suspended
+ */
+int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
+ enum kbase_pm_suspend_handler suspend_handler);
+
+
+/**
+ * kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * This function handles a stop event for the VM.
+ * It will update the VM state and forward the stop event to the driver.
+ */
+void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev);
+
+#endif /*_MALI_KBASE_ARBITER_PM_H_ */
diff --git a/mali_kbase/backend/gpu/mali_kbase_device_hw.c b/mali_kbase/backend/gpu/mali_kbase_device_hw.c
index 0ec8cef..8bdc526 100644
--- a/mali_kbase/backend/gpu/mali_kbase_device_hw.c
+++ b/mali_kbase/backend/gpu/mali_kbase_device_hw.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -190,6 +190,15 @@ u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset)
}
KBASE_EXPORT_TEST_API(kbase_reg_read);
+
+bool kbase_is_gpu_lost(struct kbase_device *kbdev)
+{
+ u32 val;
+
+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
+
+ return val == 0;
+}
#endif /* !defined(CONFIG_MALI_NO_MALI) */
/**
@@ -245,7 +254,7 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev)
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
irq_mask | CLEAN_CACHES_COMPLETED);
- KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
@@ -280,7 +289,7 @@ static void kbase_clean_caches_done(struct kbase_device *kbdev)
if (kbdev->cache_clean_queued) {
kbdev->cache_clean_queued = false;
- KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
} else {
@@ -333,7 +342,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
{
bool clear_gpu_fault = false;
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val);
if (val & GPU_FAULT)
clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev,
val & MULTIPLE_GPU_FAULTS);
@@ -344,7 +353,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
if (val & PRFCNT_SAMPLE_COMPLETED)
kbase_instr_hwcnt_sample_done(kbdev);
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val);
/* kbase_pm_check_transitions (called by kbase_pm_power_changed) must
@@ -374,5 +383,5 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
}
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val);
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_device_internal.h b/mali_kbase/backend/gpu/mali_kbase_device_internal.h
index 5ddc4a5..2e1d011 100644
--- a/mali_kbase/backend/gpu/mali_kbase_device_internal.h
+++ b/mali_kbase/backend/gpu/mali_kbase_device_internal.h
@@ -51,6 +51,18 @@ void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value);
u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset);
/**
+ * kbase_is_gpu_lost() - Has the GPU been lost.
+ * @kbdev: Kbase device pointer
+ *
+ * This function will return true if the GPU has been lost.
+ * When this happens register reads will be zero. A zero GPU_ID is
+ * invalid so this is used to detect GPU_LOST
+ *
+ * Return: True if GPU LOST
+ */
+bool kbase_is_gpu_lost(struct kbase_device *kbdev);
+
+/**
* kbase_gpu_start_cache_clean - Start a cache clean
* @kbdev: Kbase device
*
diff --git a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c
index ae3ed9c..352afa1 100644
--- a/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c
+++ b/mali_kbase/backend/gpu/mali_kbase_gpuprops_backend.c
@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
*
- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -20,8 +21,6 @@
*
*/
-
-
/*
* Base kernel property query backend APIs
*/
@@ -31,79 +30,93 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <mali_kbase_hwaccess_gpuprops.h>
-void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
int i;
+ struct kbase_gpuprops_regdump registers;
/* Fill regdump with the content of the relevant registers */
- regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
+ registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
- regdump->l2_features = kbase_reg_read(kbdev,
+ registers.l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
- regdump->core_features = kbase_reg_read(kbdev,
+ registers.core_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(CORE_FEATURES));
- regdump->tiler_features = kbase_reg_read(kbdev,
+ registers.tiler_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_FEATURES));
- regdump->mem_features = kbase_reg_read(kbdev,
+ registers.mem_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MEM_FEATURES));
- regdump->mmu_features = kbase_reg_read(kbdev,
+ registers.mmu_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MMU_FEATURES));
- regdump->as_present = kbase_reg_read(kbdev,
+ registers.as_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(AS_PRESENT));
- regdump->js_present = kbase_reg_read(kbdev,
+ registers.js_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_PRESENT));
for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
- regdump->js_features[i] = kbase_reg_read(kbdev,
+ registers.js_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_FEATURES_REG(i)));
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
- regdump->texture_features[i] = kbase_reg_read(kbdev,
+ registers.texture_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
- regdump->thread_max_threads = kbase_reg_read(kbdev,
+ registers.thread_max_threads = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_THREADS));
- regdump->thread_max_workgroup_size = kbase_reg_read(kbdev,
+ registers.thread_max_workgroup_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE));
- regdump->thread_max_barrier_size = kbase_reg_read(kbdev,
+ registers.thread_max_barrier_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE));
- regdump->thread_features = kbase_reg_read(kbdev,
+ registers.thread_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_FEATURES));
- regdump->thread_tls_alloc = kbase_reg_read(kbdev,
+ registers.thread_tls_alloc = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_TLS_ALLOC));
- regdump->shader_present_lo = kbase_reg_read(kbdev,
+ registers.shader_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_LO));
- regdump->shader_present_hi = kbase_reg_read(kbdev,
+ registers.shader_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_HI));
- regdump->tiler_present_lo = kbase_reg_read(kbdev,
+ registers.tiler_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_LO));
- regdump->tiler_present_hi = kbase_reg_read(kbdev,
+ registers.tiler_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_HI));
- regdump->l2_present_lo = kbase_reg_read(kbdev,
+ registers.l2_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_LO));
- regdump->l2_present_hi = kbase_reg_read(kbdev,
+ registers.l2_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_HI));
- regdump->stack_present_lo = kbase_reg_read(kbdev,
+ registers.stack_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_LO));
- regdump->stack_present_hi = kbase_reg_read(kbdev,
+ registers.stack_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_HI));
+
+ if (!kbase_is_gpu_lost(kbdev)) {
+ *regdump = registers;
+ return 0;
+ } else
+ return -EIO;
}
-void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) {
+ u32 coherency_features;
+
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
- regdump->coherency_features = kbase_reg_read(kbdev,
+ coherency_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(COHERENCY_FEATURES));
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
+ regdump->coherency_features = coherency_features;
+
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
} else {
@@ -112,13 +125,22 @@ void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
COHERENCY_FEATURE_BIT(COHERENCY_NONE) |
COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE);
}
+
+ return 0;
}
-void kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
- regdump->l2_features = kbase_reg_read(kbdev,
+ u32 l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
+
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
+ regdump->l2_features = l2_features;
}
+
+ return 0;
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c
index cb3e1d3..8b320c7 100644
--- a/mali_kbase/backend/gpu/mali_kbase_instr_backend.c
+++ b/mali_kbase/backend/gpu/mali_kbase_instr_backend.c
@@ -212,8 +212,8 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
kbdev->hwcnt.addr >> 32);
/* Start dumping */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL,
- kbdev->hwcnt.addr, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL,
+ kbdev->hwcnt.addr);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_SAMPLE);
@@ -359,7 +359,7 @@ int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
goto out;
/* Clear the counters */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_CLEAR);
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_as.c b/mali_kbase/backend/gpu/mali_kbase_jm_as.c
index 5b30e93..bb4f548 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_as.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_as.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -148,8 +148,7 @@ int kbase_backend_find_and_release_free_address_space(
*/
if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) &&
atomic_read(&as_kctx->refcount) == 1) {
- if (!kbasep_js_runpool_retain_ctx_nolock(kbdev,
- as_kctx)) {
+ if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) {
WARN(1, "Failed to retain active context\n");
spin_unlock_irqrestore(&kbdev->hwaccess_lock,
@@ -236,7 +235,7 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev,
if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) {
/* We need to retain it to keep the corresponding address space
*/
- kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ kbase_ctx_sched_retain_ctx_refcount(kctx);
}
return true;
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
index 819edaf..fa6bc83 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_hw.c
@@ -28,6 +28,7 @@
#include <mali_kbase_config.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <tl/mali_kbase_tracepoints.h>
+#include <mali_linux_trace.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_reset_gpu.h>
@@ -39,12 +40,6 @@
static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev);
-static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
- struct kbase_context *kctx)
-{
- return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT));
-}
-
static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
base_jd_core_req core_req,
int js)
@@ -129,6 +124,10 @@ static u64 select_job_chain(struct kbase_jd_atom *katom)
if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
return jc;
+ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
+
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
/* We can read a subset of renderpass state without holding
* higher-level locks (but not end_katom, for example).
@@ -259,7 +258,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx",
katom, kctx, js, jc_head);
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
+ KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
(u32)affinity);
KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx,
@@ -291,6 +290,10 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
}
#endif
+
+ trace_sysgraph_gpu(SGR_SUBMIT, kctx->id,
+ kbase_jd_atom_id(kctx, katom), js);
+
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
JS_COMMAND_START);
}
@@ -358,7 +361,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done)
KBASE_DEBUG_ASSERT(kbdev);
- KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done);
end_timestamp = ktime_get();
@@ -543,7 +546,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done)
*/
kbasep_try_reset_gpu_early_locked(kbdev);
}
- KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
}
void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
@@ -552,7 +555,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
base_jd_core_req core_reqs,
struct kbase_jd_atom *target_katom)
{
-#if KBASE_TRACE_ENABLE
+#if KBASE_KTRACE_ENABLE
u32 status_reg_before;
u64 job_in_head_before;
u32 status_reg_after;
@@ -608,7 +611,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action);
-#if KBASE_TRACE_ENABLE
+#if KBASE_KTRACE_ENABLE
status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS));
if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
struct kbase_jd_atom *head;
@@ -618,36 +621,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
head_kctx = head->kctx;
if (status_reg_before == BASE_JD_EVENT_ACTIVE)
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
- head, job_in_head_before, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js);
else
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
switch (action) {
case JS_COMMAND_SOFT_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_SOFT_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_SOFT_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
- head, head->jc, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
break;
default:
BUG();
@@ -655,36 +650,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
}
} else {
if (status_reg_before == BASE_JD_EVENT_ACTIVE)
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
- job_in_head_before, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js);
else
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
switch (action) {
case JS_COMMAND_SOFT_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
- js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js);
break;
case JS_COMMAND_SOFT_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js);
break;
case JS_COMMAND_SOFT_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
- js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
- 0, js);
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
break;
default:
BUG();
@@ -766,9 +753,9 @@ static int softstop_start_rp_nolock(
return -EPERM;
}
- if (WARN_ON(katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return -EINVAL;
+ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
if (WARN_ON(rp->state != KBASE_JD_RP_START &&
@@ -1058,7 +1045,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
KBASE_RESET_GPU_SILENT)
silent = true;
- KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
/* Disable GPU hardware counters.
* This call will block until counters are disabled.
@@ -1186,7 +1173,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
dev_err(kbdev->dev, "Reset complete");
/* Try submitting some jobs to restart processing */
- KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
kbase_js_sched_all(kbdev);
/* Process any pending slot updates */
@@ -1201,7 +1188,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
}
static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h
index d1ed42d..1419b59 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_internal.h
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_internal.h
@@ -34,6 +34,7 @@
#include <linux/atomic.h>
#include <backend/gpu/mali_kbase_jm_rb.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
/**
* kbase_job_submit_nolock() - Submit a job to a certain job-slot
@@ -70,6 +71,13 @@ static inline char *kbasep_make_job_slot_string(int js, char *js_string,
}
#endif
+static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
+ struct kbase_context *kctx)
+{
+ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT));
+}
+
+
/**
* kbase_job_hw_submit() - Submit a job to the GPU
* @kbdev: Device pointer
diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
index 6daea01..ec7bcb1 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
@@ -614,11 +614,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
return -EAGAIN;
}
- if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
- kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
+ if (kbase_pm_get_ready_cores(kbdev,
+ KBASE_PM_CORE_L2) ||
+ kbase_pm_get_trans_cores(kbdev,
+ KBASE_PM_CORE_L2) ||
+ kbase_is_gpu_lost(kbdev)) {
/*
- * The L2 is still powered, wait for all the users to
- * finish with it before doing the actual reset.
+ * The L2 is still powered, wait for all
+ * the users to finish with it before doing
+ * the actual reset.
*/
return -EAGAIN;
}
@@ -806,7 +810,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
lockdep_assert_held(&kbdev->hwaccess_lock);
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_lost(kbdev))
+#else
if (kbase_reset_gpu_is_active(kbdev))
+#endif
return;
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
@@ -957,11 +965,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
other_slots_busy(kbdev, js))
break;
- if ((kbdev->serialize_jobs &
- KBASE_SERIALIZE_RESET) &&
- kbase_reset_gpu_is_active(kbdev))
+#ifdef CONFIG_MALI_GEM5_BUILD
+ if (!kbasep_jm_is_js_free(kbdev, js,
+ katom[idx]->kctx))
break;
-
+#endif
/* Check if this job needs the cycle counter
* enabled before submission */
if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
@@ -1124,8 +1132,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
kbase_gpu_exception_name(
completion_code));
-#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
- KBASE_TRACE_DUMP(kbdev);
+#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0
+ KBASE_KTRACE_DUMP(kbdev);
#endif
kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
@@ -1179,8 +1187,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
}
}
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
- js, completion_code);
+ KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code);
if (job_tail != 0 && job_tail != katom->jc) {
/* Some of the job has been executed */
@@ -1189,7 +1196,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
(void *)katom, job_tail);
katom->jc = job_tail;
- KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
katom, job_tail, js);
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_js_backend.c b/mali_kbase/backend/gpu/mali_kbase_js_backend.c
index c4df191..fcc0437 100644
--- a/mali_kbase/backend/gpu/mali_kbase_js_backend.c
+++ b/mali_kbase/backend/gpu/mali_kbase_js_backend.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -301,8 +301,7 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
- KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
- 0u);
+ KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u);
}
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c
index 916916d..e67d12b 100644
--- a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c
+++ b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -21,7 +21,6 @@
*
*/
-
#include <mali_kbase.h>
#include <mali_kbase_bits.h>
#include <mali_kbase_config_defaults.h>
@@ -76,7 +75,7 @@ static const struct l2_mmu_config_limit limits[] = {
{KBASE_3BIT_AID_32, GENMASK(17, 15), 15} },
};
-void kbase_set_mmu_quirks(struct kbase_device *kbdev)
+int kbase_set_mmu_quirks(struct kbase_device *kbdev)
{
/* All older GPUs had 2 bits for both fields, this is a default */
struct l2_mmu_config_limit limit = {
@@ -101,18 +100,23 @@ void kbase_set_mmu_quirks(struct kbase_device *kbdev)
mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG));
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
mmu_config &= ~(limit.read.mask | limit.write.mask);
/* Can't use FIELD_PREP() macro here as the mask isn't constant */
mmu_config |= (limit.read.value << limit.read.shift) |
- (limit.write.value << limit.write.shift);
+ (limit.write.value << limit.write.shift);
kbdev->hw_quirks_mmu = mmu_config;
if (kbdev->system_coherency == COHERENCY_ACE) {
/* Allow memory configuration disparity to be ignored,
- * we optimize the use of shared memory and thus we
- * expect some disparity in the memory configuration.
- */
+ * we optimize the use of shared memory and thus we
+ * expect some disparity in the memory configuration.
+ */
kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY;
}
+
+ return 0;
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h
index 25636ee..0c779ac 100644
--- a/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h
+++ b/mali_kbase/backend/gpu/mali_kbase_l2_mmu_config.h
@@ -20,13 +20,22 @@
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
*/
#ifndef _KBASE_L2_MMU_CONFIG_H_
@@ -38,7 +47,9 @@
*
* Use this function to initialise the hw_quirks_mmu field, for instance to set
* the MAX_READS and MAX_WRITES to sane defaults for each GPU.
+ *
+ * Return: Zero for succeess or a Linux error code
*/
-void kbase_set_mmu_quirks(struct kbase_device *kbdev);
+int kbase_set_mmu_quirks(struct kbase_device *kbdev);
#endif /* _KBASE_L2_MMU_CONFIG_H */
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c
index 6a9cb13..576c9f2 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_backend.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_backend.c
@@ -37,6 +37,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_devfreq.h>
#include <mali_kbase_dummy_job_wa.h>
+#include <mali_kbase_irq_internal.h>
static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data);
static void kbase_pm_hwcnt_disable_worker(struct work_struct *data);
@@ -139,6 +140,9 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
kbdev->pm.backend.ca_cores_enabled = ~0ull;
kbdev->pm.backend.gpu_powered = false;
kbdev->pm.suspending = false;
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbdev->pm.gpu_lost = false;
+#endif
#ifdef CONFIG_MALI_DEBUG
kbdev->pm.backend.driver_ready_for_irqs = false;
#endif /* CONFIG_MALI_DEBUG */
@@ -245,7 +249,6 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
pm.backend.gpu_poweroff_wait_work);
struct kbase_pm_device_data *pm = &kbdev->pm;
struct kbase_pm_backend_data *backend = &pm->backend;
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long flags;
/* Wait for power transitions to complete. We do this with no locks held
@@ -253,8 +256,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
*/
kbase_pm_wait_for_desired_state(kbdev);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
if (!backend->poweron_required) {
unsigned long flags;
@@ -272,11 +274,9 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
* process. Interrupts are disabled so no more faults
* should be generated at this point.
*/
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
kbase_flush_mmu_wqs(kbdev);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
/* Turn off clock now that fault have been handled. We
* dropped locks so poweron_required may have changed -
@@ -302,8 +302,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
wake_up(&kbdev->pm.backend.poweroff_wait);
}
@@ -518,14 +517,12 @@ KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete);
int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
unsigned int flags)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long irq_flags;
int ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
@@ -534,8 +531,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
* them. */
ret = kbase_pm_init_hw(kbdev, flags);
if (ret) {
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
return ret;
}
@@ -565,8 +561,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev, false);
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
return 0;
}
@@ -654,20 +649,16 @@ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
-
/* Force power off the GPU and all cores (regardless of policy), only
* after the PM active count reaches zero (otherwise, we risk turning it
* off prematurely) */
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
kbase_pm_do_poweroff(kbdev);
kbase_backend_timer_suspend(kbdev);
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
kbase_pm_wait_for_poweroff_complete(kbdev);
@@ -677,16 +668,80 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
-
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
kbdev->pm.suspending = false;
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbdev->pm.gpu_lost = false;
+#endif
kbase_pm_do_poweron(kbdev, true);
kbase_backend_timer_resume(kbdev);
+ kbase_pm_unlock(kbdev);
+}
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
+ ktime_t end_timestamp = ktime_get();
+
+ /* Full GPU reset will have been done by hypervisor, so cancel */
+ atomic_set(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_NOT_PENDING);
+ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
+
+ /* GPU is no longer mapped to VM. So no interrupts will be received
+ * and Mali registers have been replaced by dummy RAM
+ */
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ spin_lock(&kbdev->mmu_mask_change);
+ kbdev->irq_reset_flush = true;
+ spin_unlock(&kbdev->mmu_mask_change);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ kbase_synchronize_irqs(kbdev);
+ kbase_flush_mmu_wqs(kbdev);
+ kbdev->irq_reset_flush = false;
+
+ /* Clear all jobs running on the GPU */
+ mutex_lock(&kbdev->pm.lock);
+ kbdev->pm.gpu_lost = true;
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ kbdev->protected_mode = false;
+ if (!kbdev->pm.backend.protected_entry_transition_override)
+ kbase_backend_reset(kbdev, &end_timestamp);
+ kbase_pm_metrics_update(kbdev, NULL);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+ /* Cancel any pending HWC dumps */
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Wait for all threads keeping GPU active to complete */
+ mutex_unlock(&kbdev->pm.lock);
+ wait_event(kbdev->pm.zero_active_count_wait,
+ kbdev->pm.active_count == 0);
+ mutex_lock(&kbdev->pm.lock);
+
+ /* Update state to GPU off */
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ kbdev->pm.backend.shaders_desired = false;
+ kbdev->pm.backend.l2_desired = false;
+ backend->l2_state = KBASE_L2_OFF;
+ backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF;
+ kbdev->pm.backend.gpu_powered = false;
+ backend->poweroff_wait_in_progress = false;
+ KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
+ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
+ kbase_gpu_cache_clean_wait_complete(kbdev);
+ backend->poweroff_wait_in_progress = false;
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ wake_up(&kbdev->pm.backend.poweroff_wait);
mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
}
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c
index b691524..e7eef26 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_ca.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_ca.c
@@ -66,7 +66,7 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
}
if (kbase_dummy_job_wa_enabled(kbdev)) {
- dev_info_once(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled");
+ dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled");
goto unlock;
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
index b04d705..6b821f7 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_driver.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
@@ -20,8 +21,6 @@
*
*/
-
-
/*
* Base kernel Power Management hardware control
*/
@@ -43,6 +42,9 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_l2_mmu_config.h>
#include <mali_kbase_dummy_job_wa.h>
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include <arbiter/mali_kbase_arbiter_pm.h>
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
#include <linux/of.h>
@@ -281,16 +283,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev,
if (action == ACTION_PWRON)
switch (core_type) {
case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
- lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores);
break;
case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
- NULL, 0u, lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores);
break;
case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
- 0u, lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores);
break;
default:
break;
@@ -298,16 +297,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev,
else if (action == ACTION_PWROFF)
switch (core_type) {
case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
- 0u, lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores);
break;
case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
- NULL, 0u, lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores);
break;
case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
- 0u, lo);
+ KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores);
/* disable snoops before L2 is turned off */
kbase_pm_cache_snoop_disable(kbdev);
break;
@@ -444,16 +440,13 @@ u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
switch (type) {
case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
- (u32) result);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result);
break;
case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
- (u32) result);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result);
break;
case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
- (u32) result);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result);
break;
default:
break;
@@ -543,7 +536,7 @@ static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state)
return strings[state];
}
-static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
+static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
{
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present;
@@ -563,6 +556,13 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
u64 tiler_ready = kbase_pm_get_ready_cores(kbdev,
KBASE_PM_CORE_TILER);
+ /*
+ * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
+ * are vulnerable to corruption if gpu is lost
+ */
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
/* mask off ready from trans in case transitions finished
* between the register reads
*/
@@ -600,10 +600,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
case KBASE_L2_PEND_ON:
if (!l2_trans && l2_ready == l2_present && !tiler_trans
&& tiler_ready == tiler_present) {
- KBASE_TRACE_ADD(kbdev,
- PM_CORES_CHANGE_AVAILABLE_TILER,
- NULL, NULL, 0u,
- (u32)tiler_ready);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready);
/*
* Ensure snoops are enabled after L2 is powered
* up. Note that kbase keeps track of the snoop
@@ -773,8 +770,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
kbase_gpu_start_cache_clean_nolock(
kbdev);
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
- NULL, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
backend->l2_state = KBASE_L2_PEND_OFF;
break;
@@ -819,8 +815,6 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
&kbdev->pm.backend.gpu_poweroff_wait_work);
}
- if (backend->l2_state == KBASE_L2_ON)
- return l2_present;
return 0;
}
@@ -897,13 +891,14 @@ static const char *kbase_shader_core_state_to_string(
return strings[state];
}
-static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
+static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
{
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
struct kbasep_pm_tick_timer_state *stt =
&kbdev->pm.backend.shader_tick_timer;
enum kbase_shader_core_state prev_state;
u64 stacks_avail = 0;
+ int err = 0;
lockdep_assert_held(&kbdev->hwaccess_lock);
@@ -925,6 +920,15 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK);
}
+ /*
+ * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
+ * are vulnerable to corruption if gpu is lost
+ */
+ if (kbase_is_gpu_lost(kbdev)) {
+ err = -EIO;
+ break;
+ }
+
/* mask off ready from trans in case transitions finished
* between the register reads
*/
@@ -978,9 +982,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
case KBASE_SHADERS_PEND_ON_CORESTACK_ON:
if (!shaders_trans && shaders_ready == backend->shaders_avail) {
- KBASE_TRACE_ADD(kbdev,
- PM_CORES_CHANGE_AVAILABLE,
- NULL, NULL, 0u, (u32)shaders_ready);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready);
backend->pm_shaders_core_mask = shaders_ready;
backend->hwcnt_desired = true;
if (backend->hwcnt_disabled) {
@@ -1017,6 +1019,10 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
;
} else if (!backend->shaders_desired) {
if (kbdev->pm.backend.protected_transition_override ||
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbase_pm_is_suspending(kbdev) ||
+ kbase_pm_is_gpu_lost(kbdev) ||
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
!stt->configured_ticks ||
WARN_ON(stt->cancel_queued)) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
@@ -1076,6 +1082,11 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK;
} else if (stt->remaining_ticks == 0) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ } else if (kbase_pm_is_suspending(kbdev) ||
+ kbase_pm_is_gpu_lost(kbdev)) {
+ backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
}
break;
@@ -1129,17 +1140,12 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
shaders_ready & ~backend->shaders_avail, ACTION_PWROFF);
backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON;
- KBASE_TRACE_ADD(kbdev,
- PM_CORES_CHANGE_AVAILABLE,
- NULL, NULL, 0u,
- (u32)(shaders_ready & ~backend->shaders_avail));
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail));
} else {
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
shaders_ready, ACTION_PWROFF);
- KBASE_TRACE_ADD(kbdev,
- PM_CORES_CHANGE_AVAILABLE,
- NULL, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON;
}
@@ -1188,6 +1194,8 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
backend->shaders_state));
} while (backend->shaders_state != prev_state);
+
+ return err;
}
static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev)
@@ -1280,24 +1288,29 @@ void kbase_pm_update_state(struct kbase_device *kbdev)
if (!kbdev->pm.backend.gpu_powered)
return; /* Do nothing if the GPU is off */
- kbase_pm_l2_update_state(kbdev);
- kbase_pm_shaders_update_state(kbdev);
+ if (kbase_pm_l2_update_state(kbdev))
+ return;
+
+ if (kbase_pm_shaders_update_state(kbdev))
+ return;
/* If the shaders just turned off, re-invoke the L2 state machine, in
* case it was waiting for the shaders to turn off before powering down
* the L2.
*/
if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF &&
- kbdev->pm.backend.shaders_state == KBASE_SHADERS_OFF_CORESTACK_OFF)
- kbase_pm_l2_update_state(kbdev);
+ kbdev->pm.backend.shaders_state ==
+ KBASE_SHADERS_OFF_CORESTACK_OFF) {
+ if (kbase_pm_l2_update_state(kbdev))
+ return;
+ }
if (kbase_pm_is_in_desired_state_nolock(kbdev)) {
- KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
- true, kbdev->pm.backend.shaders_avail);
+ KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail);
kbase_pm_trace_power_state(kbdev);
- KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
}
}
@@ -1576,7 +1589,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
kbdev->poweroff_pending = false;
- KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u);
if (is_resume && kbdev->pm.backend.callback_power_resume) {
kbdev->pm.backend.callback_power_resume(kbdev);
@@ -1639,7 +1652,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
return true;
}
- KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u);
/* Disable interrupts. This also clears any outstanding interrupts */
kbase_pm_disable_interrupts(kbdev);
@@ -1660,6 +1673,9 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
/* The GPU power may be turned off from this point */
kbdev->pm.backend.gpu_powered = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbdev->pm.backend.callback_power_off)
kbdev->pm.backend.callback_power_off(kbdev);
@@ -1713,9 +1729,9 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
-static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
+static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
{
- kbdev->hw_quirks_jm = kbase_reg_read(kbdev,
+ u32 hw_quirks_jm = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JM_CONFIG));
if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) {
@@ -1730,11 +1746,17 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
*/
if (coherency_features ==
COHERENCY_FEATURE_BIT(COHERENCY_ACE)) {
- kbdev->hw_quirks_jm |= (COHERENCY_ACE_LITE |
+ hw_quirks_jm |= (COHERENCY_ACE_LITE |
COHERENCY_ACE) <<
JM_FORCE_COHERENCY_FEATURES_SHIFT;
}
}
+
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
+ kbdev->hw_quirks_jm = hw_quirks_jm;
+
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) {
int default_idvs_group_size = 0xF;
u32 tmp;
@@ -1756,40 +1778,58 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
#define MANUAL_POWER_CONTROL ((u32)(1 << 8))
if (corestack_driver_control)
kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL;
+
+ return 0;
}
-static void kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id)
+static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id)
{
- kbdev->hw_quirks_sc = kbase_reg_read(kbdev,
+ u32 hw_quirks_sc = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_CONFIG));
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */
- kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
+ hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */
- kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
+ hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162))
- kbdev->hw_quirks_sc |= SC_VAR_ALGORITHM;
+ hw_quirks_sc |= SC_VAR_ALGORITHM;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING))
- kbdev->hw_quirks_sc |= SC_TLS_HASH_ENABLE;
+ hw_quirks_sc |= SC_TLS_HASH_ENABLE;
+
+ kbdev->hw_quirks_sc = hw_quirks_sc;
+
+ return 0;
}
-static void kbase_set_tiler_quirks(struct kbase_device *kbdev)
+static int kbase_set_tiler_quirks(struct kbase_device *kbdev)
{
- kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
+ u32 hw_quirks_tiler = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_CONFIG));
+
+ if (kbase_is_gpu_lost(kbdev))
+ return -EIO;
+
/* Set tiler clock gate override if required */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
- kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
+ hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
+
+ kbdev->hw_quirks_tiler = hw_quirks_tiler;
+
+ return 0;
}
-static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
+static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
{
struct device_node *np = kbdev->dev->of_node;
const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
GPU_ID_VERSION_PRODUCT_ID_SHIFT;
+ int error = 0;
kbdev->hw_quirks_jm = 0;
kbdev->hw_quirks_sc = 0;
@@ -1802,7 +1842,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_jm = [0x%x] in Devicetree\n",
kbdev->hw_quirks_jm);
} else {
- kbase_set_jm_quirks(kbdev, prod_id);
+ error = kbase_set_jm_quirks(kbdev, prod_id);
+ if (error)
+ return error;
}
if (!of_property_read_u32(np, "quirks_sc",
@@ -1811,7 +1853,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_sc = [0x%x] in Devicetree\n",
kbdev->hw_quirks_sc);
} else {
- kbase_set_sc_quirks(kbdev, prod_id);
+ error = kbase_set_sc_quirks(kbdev, prod_id);
+ if (error)
+ return error;
}
if (!of_property_read_u32(np, "quirks_tiler",
@@ -1820,7 +1864,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_tiler = [0x%x] in Devicetree\n",
kbdev->hw_quirks_tiler);
} else {
- kbase_set_tiler_quirks(kbdev);
+ error = kbase_set_tiler_quirks(kbdev);
+ if (error)
+ return error;
}
if (!of_property_read_u32(np, "quirks_mmu",
@@ -1829,8 +1875,10 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_mmu = [0x%x] in Devicetree\n",
kbdev->hw_quirks_mmu);
} else {
- kbase_set_mmu_quirks(kbdev);
+ error = kbase_set_mmu_quirks(kbdev);
}
+
+ return error;
}
static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
@@ -1892,7 +1940,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
struct kbasep_reset_timeout_data rtdata;
int ret;
- KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0);
KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev);
@@ -1933,8 +1981,9 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
/* No interrupt has been received - check if the RAWSTAT register says
* the reset has completed */
- if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) &
- RESET_COMPLETED) {
+ if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) &
+ RESET_COMPLETED)
+ || kbase_is_gpu_lost(kbdev)) {
/* The interrupt is set in the RAWSTAT; this suggests that the
* interrupts are not getting to the CPU */
dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
@@ -1947,7 +1996,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
* reset */
dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
RESET_TIMEOUT);
- KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_HARD_RESET);
@@ -2016,8 +2065,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
/* The cores should be made unavailable due to the reset */
spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
- NULL, 0u, (u32)0u);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
/* Soft reset the GPU */
@@ -2031,8 +2079,11 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
if (err)
goto exit;
- if (flags & PM_HW_ISSUES_DETECT)
- kbase_pm_hw_issues_detect(kbdev);
+ if (flags & PM_HW_ISSUES_DETECT) {
+ err = kbase_pm_hw_issues_detect(kbdev);
+ if (err)
+ goto exit;
+ }
kbase_pm_hw_issues_apply(kbdev);
kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
index 8d19318..95f10e0 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_internal.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -682,4 +682,29 @@ extern bool corestack_driver_control;
* Return: true if l2 need to power on
*/
bool kbase_pm_is_l2_desired(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_lock - Lock all necessary mutexes to perform PM actions
+ *
+ * @kbdev: Device pointer
+ *
+ * This function locks correct mutexes independent of GPU architecture.
+ */
+static inline void kbase_pm_lock(struct kbase_device *kbdev)
+{
+ mutex_lock(&kbdev->js_data.runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+}
+
+/**
+ * kbase_pm_unlock - Unlock mutexes locked by kbase_pm_lock
+ *
+ * @kbdev: Device pointer
+ */
+static inline void kbase_pm_unlock(struct kbase_device *kbdev)
+{
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&kbdev->js_data.runpool_mutex);
+}
+
#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
index ae494b0..de3babe 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_metrics.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -31,6 +31,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_jm_rb.h>
#include <backend/gpu/mali_kbase_pm_defs.h>
+#include <mali_linux_trace.h>
/* When VSync is being hit aim for utilisation between 70-90% */
#define KBASE_PM_VSYNC_MIN_UTILISATION 70
@@ -284,8 +285,11 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev)
active_cl_ctx[device_nr] = 1;
} else {
kbdev->pm.backend.metrics.active_gl_ctx[js] = 1;
+ trace_sysgraph(SGR_ACTIVE, 0, js);
}
kbdev->pm.backend.metrics.gpu_active = true;
+ } else {
+ trace_sysgraph(SGR_INACTIVE, 0, js);
}
}
}
diff --git a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c
index 7d9bb03..17ed21e 100644
--- a/mali_kbase/backend/gpu/mali_kbase_pm_policy.c
+++ b/mali_kbase/backend/gpu/mali_kbase_pm_policy.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -102,6 +102,8 @@ void kbase_pm_update_active(struct kbase_device *kbdev)
* when there are contexts active */
KBASE_DEBUG_ASSERT(pm->active_count == 0);
+ pm->backend.poweron_required = false;
+
/* Request power off */
if (pm->backend.gpu_powered) {
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@@ -157,8 +159,7 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev);
if (kbdev->pm.backend.shaders_desired != shaders_desired) {
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
- (u32)kbdev->pm.backend.shaders_desired);
+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired);
kbdev->pm.backend.shaders_desired = shaders_desired;
kbase_pm_update_state(kbdev);
@@ -199,22 +200,20 @@ KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
void kbase_pm_set_policy(struct kbase_device *kbdev,
const struct kbase_pm_policy *new_policy)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
const struct kbase_pm_policy *old_policy;
unsigned long flags;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(new_policy != NULL);
- KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
+ KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id);
/* During a policy change we pretend the GPU is active */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread */
kbase_pm_context_active(kbdev);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
/* Remove the policy to prevent IRQ handlers from working on it */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
@@ -222,13 +221,11 @@ void kbase_pm_set_policy(struct kbase_device *kbdev,
kbdev->pm.backend.pm_current_policy = NULL;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
- old_policy->id);
+ KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, old_policy->id);
if (old_policy->term)
old_policy->term(kbdev);
- KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
- new_policy->id);
+ KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id);
if (new_policy->init)
new_policy->init(kbdev);
@@ -242,8 +239,7 @@ void kbase_pm_set_policy(struct kbase_device *kbdev,
kbase_pm_update_active(kbdev);
kbase_pm_update_cores_state(kbdev);
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
/* Now the policy change is finished, we release our fake context active
* reference */
diff --git a/mali_kbase/backend/gpu/mali_kbase_time.c b/mali_kbase/backend/gpu/mali_kbase_time.c
index 057bf10..cb10518 100644
--- a/mali_kbase/backend/gpu/mali_kbase_time.c
+++ b/mali_kbase/backend/gpu/mali_kbase_time.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2016,2018-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -26,7 +26,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
- u64 *system_time, struct timespec *ts)
+ u64 *system_time, struct timespec64 *ts)
{
u32 hi1, hi2;
@@ -60,7 +60,11 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
/* Record the CPU's idea of current time */
if (ts != NULL)
- getrawmonotonic(ts);
+#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE)
+ *ts = ktime_to_timespec64(ktime_get_raw());
+#else
+ ktime_get_raw_ts64(ts);
+#endif
kbase_pm_release_gpu_cycle_counter(kbdev);
}
diff --git a/mali_kbase/build.bp b/mali_kbase/build.bp
index 94189b1..51aeecd 100644
--- a/mali_kbase/build.bp
+++ b/mali_kbase/build.bp
@@ -37,9 +37,6 @@ bob_defaults {
buslog: {
kbuild_options: ["CONFIG_MALI_BUSLOG=y"],
},
- cinstr_job_dump: {
- kbuild_options: ["CONFIG_MALI_JOB_DUMP=y"],
- },
cinstr_vector_dump: {
kbuild_options: ["CONFIG_MALI_VECTOR_DUMP=y"],
},
@@ -49,6 +46,9 @@ bob_defaults {
mali_gator_support: {
kbuild_options: ["CONFIG_MALI_GATOR_SUPPORT=y"],
},
+ mali_midgard_enable_trace: {
+ kbuild_options: ["CONFIG_MALI_MIDGARD_ENABLE_TRACE=y"],
+ },
mali_system_trace: {
kbuild_options: ["CONFIG_MALI_SYSTEM_TRACE=y"],
},
@@ -64,6 +64,12 @@ bob_defaults {
mali_dma_buf_legacy_compat: {
kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"],
},
+ mali_arbiter_support: {
+ kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"],
+ },
+ mali_gem5_build: {
+ kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"],
+ },
kbuild_options: [
"MALI_UNIT_TEST={{.unit_test_code}}",
"MALI_CUSTOMER_RELEASE={{.release}}",
@@ -92,6 +98,8 @@ bob_kernel_module {
"platform/*/*.h",
"platform/*/Kbuild",
"thirdparty/*.c",
+ "debug/*.c",
+ "debug/*.h",
"device/*.c",
"device/*.h",
"gpu/*.c",
@@ -139,6 +147,8 @@ bob_kernel_module {
gpu_has_job_manager: {
srcs: [
"context/backend/*_jm.c",
+ "debug/backend/*_jm.c",
+ "debug/backend/*_jm.h",
"device/backend/*_jm.c",
"gpu/backend/*_jm.c",
"gpu/backend/*_jm.h",
@@ -153,6 +163,8 @@ bob_kernel_module {
"csf/*.c",
"csf/*.h",
"csf/Kbuild",
+ "debug/backend/*_csf.c",
+ "debug/backend/*_csf.h",
"device/backend/*_csf.c",
"gpu/backend/*_csf.c",
"gpu/backend/*_csf.h",
@@ -160,5 +172,12 @@ bob_kernel_module {
"mmu/backend/*_csf.c",
],
},
+ mali_arbiter_support: {
+ srcs: [
+ "arbiter/*.c",
+ "arbiter/*.h",
+ "arbiter/Kbuild",
+ ],
+ },
defaults: ["mali_kbase_shared_config_defaults"],
}
diff --git a/mali_kbase/context/mali_kbase_context.c b/mali_kbase/context/mali_kbase_context.c
index a539edb..93fe431 100644
--- a/mali_kbase/context/mali_kbase_context.c
+++ b/mali_kbase/context/mali_kbase_context.c
@@ -80,7 +80,9 @@ int kbase_context_common_init(struct kbase_context *kctx)
mutex_lock(&kctx->kbdev->kctx_list_lock);
list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list);
- /* Trace with the AOM tracepoint even in CSF for dumping */
+
+ KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id,
+ kctx->kbdev->gpu_props.props.raw_props.gpu_id);
KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id,
(u32)(kctx->tgid));
mutex_unlock(&kctx->kbdev->kctx_list_lock);
@@ -107,19 +109,21 @@ void kbase_context_common_term(struct kbase_context *kctx)
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
mutex_lock(&kctx->kbdev->kctx_list_lock);
- /* Trace with the AOM tracepoint even in CSF for dumping */
+
+ KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id);
+
KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx);
list_del(&kctx->kctx_list_link);
mutex_unlock(&kctx->kbdev->kctx_list_lock);
- KBASE_TRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u);
/* Flush the timeline stream, so the user can see the termination
* tracepoints being fired.
* The "if" statement below is for optimization. It is safe to call
* kbase_timeline_streams_flush when timeline is disabled.
*/
- if (atomic_read(&kctx->kbdev->timeline_is_enabled) != 0)
+ if (atomic_read(&kctx->kbdev->timeline_flags) != 0)
kbase_timeline_streams_flush(kctx->kbdev->timeline);
vfree(kctx);
diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h
new file mode 100644
index 0000000..d534f30
--- /dev/null
+++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_codes_jm.h
@@ -0,0 +1,170 @@
+/*
+ *
+ * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/*
+ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE *****
+ * ***** DO NOT INCLUDE DIRECTLY *****
+ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL *****
+ */
+
+/*
+ * The purpose of this header file is just to contain a list of trace code
+ * identifiers
+ *
+ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT
+ * DESCRIBED IN mali_kbase_debug_ktrace_codes.h
+ */
+
+#if 0 /* Dummy section to avoid breaking formatting */
+int dummy_array[] = {
+#endif
+
+ /*
+ * Job Slot management events
+ */
+ /* info_val==irq rawstat at start */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ),
+ /* info_val==jobs processed */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END),
+ /* In the following:
+ *
+ * - ctx is set if a corresponding job found (NULL otherwise, e.g. some
+ * soft-stop cases)
+ * - uatom==kernel-side mapped uatom address (for correlation with
+ * user-side)
+ */
+ /* info_val==exit code; gpu_addr==chain gpuaddr */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE),
+ /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of
+ * affinity
+ */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT),
+ /* gpu_addr is as follows:
+ * - If JS_STATUS active after soft-stop, val==gpu addr written to
+ * JS_HEAD on submit
+ * - otherwise gpu_addr==0
+ */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1),
+ /* gpu_addr==JS_HEAD read */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP),
+ /* gpu_addr==JS_HEAD read */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0),
+ /* gpu_addr==JS_HEAD read */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1),
+ /* gpu_addr==JS_TAIL read */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD),
+ /* gpu_addr is as follows:
+ * - If JS_STATUS active before soft-stop, val==JS_HEAD
+ * - otherwise gpu_addr==0
+ */
+ /* gpu_addr==JS_HEAD read */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE),
+ /* info_val == is_scheduled */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED),
+ /* info_val == is_scheduled */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE),
+ /* info_val == nr jobs submitted */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP),
+ /* gpu_addr==JS_HEAD_NEXT last written */
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER),
+ KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER),
+ /*
+ * Job dispatch events
+ */
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB),
+ /* gpu_addr==0, info_val==0, uatom==0 */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER),
+ /*
+ * Scheduler Core events
+ */
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB),
+ /* gpu_addr==last value written/would be written to JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB),
+ /* gpu_addr==value to write into JS_HEAD */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED),
+ /* info_val == lower 32 bits of affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT),
+ /* info_val == lower 32 bits of affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED),
+ /* info_val == lower 32 bits of affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED),
+ /* info_val == lower 32 bits of rechecked affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED),
+ /* info_val == lower 32 bits of rechecked affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED),
+ /* info_val == lower 32 bits of affinity */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE),
+ /* info_val == the ctx attribute now on ctx */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX),
+ /* info_val == the ctx attribute now on runpool */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL),
+ /* info_val == the ctx attribute now off ctx */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX),
+ /* info_val == the ctx attribute now off runpool */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL),
+ /*
+ * Scheduler Policy events
+ */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX),
+ /* info_val == whether it was evicted */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ),
+ /* gpu_addr==JS_HEAD to write if the job were run */
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START),
+ KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END),
+
+#if 0 /* Dummy section to avoid breaking formatting */
+};
+#endif
+
+/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */
diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h
new file mode 100644
index 0000000..55b66ad
--- /dev/null
+++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_defs_jm.h
@@ -0,0 +1,75 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_
+#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_
+
+/**
+ * DOC: KTrace version history, JM variant
+ * 1.0:
+ * - Original version (implicit, header did not carry version information)
+ * 2.0:
+ * - Introduced version information into the header
+ * - some changes of parameter names in header
+ * - trace now uses all 64-bits of info_val
+ * - Non-JM specific parts moved to using info_val instead of refcount/gpu_addr
+ */
+#define KBASE_KTRACE_VERSION_MAJOR 2
+#define KBASE_KTRACE_VERSION_MINOR 0
+
+/* indicates if the trace message has a valid refcount member */
+#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0)
+/* indicates if the trace message has a valid jobslot member */
+#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1)
+/* indicates if the trace message has valid atom related info. */
+#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2)
+
+
+/**
+ * struct kbase_ktrace_backend - backend specific part of a trace message
+ *
+ * @atom_udata: Copy of the user data sent for the atom in base_jd_submit.
+ * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags
+ * @gpu_addr: GPU address, usually of the job-chain represented by an atom.
+ * @atom_number: id of the atom for which trace message was added. Only valid
+ * if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags
+ * @code: Identifies the event, refer to enum kbase_ktrace_code.
+ * @flags: indicates information about the trace message itself. Used
+ * during dumping of the message.
+ * @jobslot: job-slot for which trace message was added, valid only for
+ * job-slot management events.
+ * @refcount: reference count for the context, valid for certain events
+ * related to scheduler core and policy.
+ */
+struct kbase_ktrace_backend {
+ /* Place 64 and 32-bit members together */
+ u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */
+ u64 gpu_addr;
+ int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */
+ /* Pack smaller members together */
+ kbase_ktrace_code_t code;
+ kbase_ktrace_flag_t flags;
+ u8 jobslot;
+ u8 refcount;
+};
+
+#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */
diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c
new file mode 100644
index 0000000..e651a09
--- /dev/null
+++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.c
@@ -0,0 +1,113 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+#include <mali_kbase.h>
+#include "debug/mali_kbase_debug_ktrace_internal.h"
+#include "debug/backend/mali_kbase_debug_ktrace_jm.h"
+
+#if KBASE_KTRACE_TARGET_RBUF
+
+void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written)
+{
+ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0),
+ "katom,gpu_addr,jobslot,refcount"), 0);
+}
+
+void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg,
+ char *buffer, int sz, s32 *written)
+{
+ /* katom */
+ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM)
+ *written += MAX(snprintf(buffer + *written,
+ MAX(sz - *written, 0),
+ "atom %d (ud: 0x%llx 0x%llx)",
+ trace_msg->backend.atom_number,
+ trace_msg->backend.atom_udata[0],
+ trace_msg->backend.atom_udata[1]), 0);
+
+ /* gpu_addr */
+ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND)
+ *written += MAX(snprintf(buffer + *written,
+ MAX(sz - *written, 0),
+ ",%.8llx,", trace_msg->backend.gpu_addr), 0);
+ else
+ *written += MAX(snprintf(buffer + *written,
+ MAX(sz - *written, 0),
+ ",,"), 0);
+
+ /* jobslot */
+ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT)
+ *written += MAX(snprintf(buffer + *written,
+ MAX(sz - *written, 0),
+ "%d", trace_msg->backend.jobslot), 0);
+
+ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0),
+ ","), 0);
+
+ /* refcount */
+ if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT)
+ *written += MAX(snprintf(buffer + *written,
+ MAX(sz - *written, 0),
+ "%d", trace_msg->backend.refcount), 0);
+}
+
+void kbasep_ktrace_add_jm(struct kbase_device *kbdev,
+ enum kbase_ktrace_code code, struct kbase_context *kctx,
+ struct kbase_jd_atom *katom, u64 gpu_addr,
+ kbase_ktrace_flag_t flags, int refcount, int jobslot,
+ u64 info_val)
+{
+ unsigned long irqflags;
+ struct kbase_ktrace_msg *trace_msg;
+
+ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags);
+
+ /* Reserve and update indices */
+ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace);
+
+ /* Fill the common part of the message (including backend.flags) */
+ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags,
+ info_val);
+
+ /* Indicate to the common code that backend-specific parts will be
+ * valid
+ */
+ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND;
+
+ /* Fill the JM-specific parts of the message */
+ if (katom) {
+ trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM;
+
+ trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom);
+ trace_msg->backend.atom_udata[0] = katom->udata.blob[0];
+ trace_msg->backend.atom_udata[1] = katom->udata.blob[1];
+ }
+
+ trace_msg->backend.gpu_addr = gpu_addr;
+ trace_msg->backend.jobslot = jobslot;
+ /* Clamp refcount */
+ trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF);
+
+ /* Done */
+ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags);
+}
+
+#endif /* KBASE_KTRACE_TARGET_RBUF */
diff --git a/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h
new file mode 100644
index 0000000..c1bacf9
--- /dev/null
+++ b/mali_kbase/debug/backend/mali_kbase_debug_ktrace_jm.h
@@ -0,0 +1,362 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+#ifndef _KBASE_DEBUG_KTRACE_JM_H_
+#define _KBASE_DEBUG_KTRACE_JM_H_
+
+/*
+ * KTrace target for internal ringbuffer
+ */
+#if KBASE_KTRACE_TARGET_RBUF
+/**
+ * kbasep_ktrace_add_jm - internal function to add trace about Job Management
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @flags: flags about the message
+ * @refcount: reference count information to add to the trace
+ * @jobslot: jobslot information to add to the trace
+ * @info_val: generic information about @code to add to the trace
+ *
+ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead.
+ */
+void kbasep_ktrace_add_jm(struct kbase_device *kbdev,
+ enum kbase_ktrace_code code, struct kbase_context *kctx,
+ struct kbase_jd_atom *katom, u64 gpu_addr,
+ kbase_ktrace_flag_t flags, int refcount, int jobslot,
+ u64 info_val);
+
+#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot) \
+ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
+ gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, 0)
+
+#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot, info_val) \
+ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
+ gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, \
+ info_val)
+
+#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
+ refcount) \
+ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
+ gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, 0)
+#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, refcount, info_val) \
+ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
+ gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, \
+ info_val)
+
+#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \
+ kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
+ gpu_addr, 0, 0, 0, info_val)
+
+#else /* KBASE_KTRACE_TARGET_RBUF */
+#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(jobslot);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot, info_val) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(jobslot);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
+ refcount) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(refcount);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, refcount, info_val) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
+ info_val)\
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+
+/*
+ * KTrace target for Linux's ftrace
+ */
+#if KBASE_KTRACE_TARGET_FTRACE
+#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot) \
+ trace_mali_##code(jobslot, 0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, jobslot, info_val) \
+ trace_mali_##code(jobslot, info_val)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \
+ gpu_addr, refcount) \
+ trace_mali_##code(refcount, 0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, refcount, info_val) \
+ trace_mali_##code(refcount, info_val)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
+ info_val) \
+ trace_mali_##code(gpu_addr, info_val)
+#else /* KBASE_KTRACE_TARGET_FTRACE */
+#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(jobslot);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, jobslot, info_val) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(jobslot);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \
+ gpu_addr, refcount) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(refcount);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, refcount, info_val) \
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+
+#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
+ info_val)\
+ do {\
+ CSTD_UNUSED(kbdev);\
+ CSTD_NOP(code);\
+ CSTD_UNUSED(kctx);\
+ CSTD_UNUSED(katom);\
+ CSTD_UNUSED(gpu_addr);\
+ CSTD_UNUSED(info_val);\
+ CSTD_NOP(0);\
+ } while (0)
+#endif /* KBASE_KTRACE_TARGET_FTRACE */
+
+/*
+ * Master set of macros to route KTrace to any of the targets
+ */
+
+/**
+ * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @jobslot: jobslot information to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __gpu_addr = gpu_addr; \
+ int __jobslot = jobslot; \
+ KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \
+ KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @jobslot: jobslot information to add to the trace
+ * @info_val: generic information about @code to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
+ jobslot, info_val) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __gpu_addr = gpu_addr; \
+ int __jobslot = jobslot; \
+ u64 __info_val = info_val; \
+ KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \
+ KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @refcount: reference count information to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
+ refcount) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __gpu_addr = gpu_addr; \
+ int __refcount = refcount; \
+ KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \
+ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount,
+ * and info
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @refcount: reference count information to add to the trace
+ * @info_val: generic information about @code to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
+ gpu_addr, refcount, info_val) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __gpu_addr = gpu_addr; \
+ int __refcount = refcount; \
+ u64 __info_val = info_val; \
+ KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \
+ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount)
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @katom: kbase atom, or NULL if no atom
+ * @gpu_addr: GPU address, usually related to @katom
+ * @info_val: generic information about @code to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __gpu_addr = gpu_addr; \
+ u64 __info_val = info_val; \
+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \
+ KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \
+ } while (0)
+
+#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */
diff --git a/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h
new file mode 100644
index 0000000..d964e5a
--- /dev/null
+++ b/mali_kbase/debug/backend/mali_kbase_debug_linux_ktrace_jm.h
@@ -0,0 +1,150 @@
+/*
+ *
+ * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/*
+ * NOTE: This must **only** be included through mali_linux_trace.h,
+ * otherwise it will fail to setup tracepoints correctly
+ */
+
+#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_
+
+DECLARE_EVENT_CLASS(mali_jm_slot_template,
+ TP_PROTO(int jobslot, u64 info_val),
+ TP_ARGS(jobslot, info_val),
+ TP_STRUCT__entry(
+ __field(unsigned int, jobslot)
+ __field(u64, info_val)
+ ),
+ TP_fast_assign(
+ __entry->jobslot = jobslot;
+ __entry->info_val = info_val;
+ ),
+ TP_printk("jobslot=%u info=0x%llx", __entry->jobslot, __entry->info_val)
+);
+
+#define DEFINE_MALI_JM_SLOT_EVENT(name) \
+DEFINE_EVENT(mali_jm_slot_template, mali_##name, \
+ TP_PROTO(int jobslot, u64 info_val), \
+ TP_ARGS(jobslot, info_val))
+DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT);
+DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE);
+DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD);
+DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD);
+DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP);
+DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0);
+DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1);
+DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP);
+DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0);
+DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1);
+DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP);
+DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT);
+DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER);
+DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER);
+DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT);
+DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB);
+DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE);
+DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB);
+DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED);
+DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB);
+DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ);
+#undef DEFINE_MALI_JM_SLOT_EVENT
+
+DECLARE_EVENT_CLASS(mali_jm_refcount_template,
+ TP_PROTO(int refcount, u64 info_val),
+ TP_ARGS(refcount, info_val),
+ TP_STRUCT__entry(
+ __field(unsigned int, refcount)
+ __field(u64, info_val)
+ ),
+ TP_fast_assign(
+ __entry->refcount = refcount;
+ __entry->info_val = info_val;
+ ),
+ TP_printk("refcount=%u info=0x%llx", __entry->refcount, __entry->info_val)
+);
+
+#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \
+DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \
+ TP_PROTO(int refcount, u64 info_val), \
+ TP_ARGS(refcount, info_val))
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX);
+DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS);
+#undef DEFINE_MALI_JM_REFCOUNT_EVENT
+
+DECLARE_EVENT_CLASS(mali_jm_add_template,
+ TP_PROTO(u64 gpu_addr, u64 info_val),
+ TP_ARGS(gpu_addr, info_val),
+ TP_STRUCT__entry(
+ __field(u64, gpu_addr)
+ __field(u64, info_val)
+ ),
+ TP_fast_assign(
+ __entry->gpu_addr = gpu_addr;
+ __entry->info_val = info_val;
+ ),
+ TP_printk("gpu_addr=0x%llx info=0x%llx", __entry->gpu_addr, __entry->info_val)
+);
+
+#define DEFINE_MALI_JM_ADD_EVENT(name) \
+DEFINE_EVENT(mali_jm_add_template, mali_##name, \
+ TP_PROTO(u64 gpu_addr, u64 info_val), \
+ TP_ARGS(gpu_addr, info_val))
+DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER);
+DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END);
+DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER);
+DEFINE_MALI_JM_ADD_EVENT(JD_DONE);
+DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL);
+DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT);
+DEFINE_MALI_JM_ADD_EVENT(JM_IRQ);
+DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END);
+DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS);
+DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE);
+DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED);
+DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED);
+DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE);
+DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET);
+DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE);
+DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL);
+DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL);
+DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX);
+DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX);
+DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END);
+DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START);
+DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB);
+#undef DEFINE_MALI_JM_ADD_EVENT
+
+#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/
diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.c b/mali_kbase/debug/mali_kbase_debug_ktrace.c
new file mode 100644
index 0000000..6322abb
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_ktrace.c
@@ -0,0 +1,342 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+#include <mali_kbase.h>
+#include "debug/mali_kbase_debug_ktrace_internal.h"
+
+int kbase_ktrace_init(struct kbase_device *kbdev)
+{
+#if KBASE_KTRACE_TARGET_RBUF
+ struct kbase_ktrace_msg *rbuf;
+
+ /* See also documentation of enum kbase_ktrace_code */
+ compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) ||
+ KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)),
+ "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT");
+
+ rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL);
+
+ if (!rbuf)
+ return -EINVAL;
+
+ kbdev->ktrace.rbuf = rbuf;
+ spin_lock_init(&kbdev->ktrace.lock);
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+ return 0;
+}
+
+void kbase_ktrace_term(struct kbase_device *kbdev)
+{
+#if KBASE_KTRACE_TARGET_RBUF
+ kfree(kbdev->ktrace.rbuf);
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+}
+
+void kbase_ktrace_hook_wrapper(void *param)
+{
+ struct kbase_device *kbdev = (struct kbase_device *)param;
+
+ KBASE_KTRACE_DUMP(kbdev);
+}
+
+#if KBASE_KTRACE_TARGET_RBUF
+
+static const char * const kbasep_ktrace_code_string[] = {
+ /*
+ * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
+ * THIS MUST BE USED AT THE START OF THE ARRAY
+ */
+#define KBASE_KTRACE_CODE_MAKE_CODE(X) # X
+#include "debug/mali_kbase_debug_ktrace_codes.h"
+#undef KBASE_KTRACE_CODE_MAKE_CODE
+};
+
+static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written)
+{
+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
+ "secs,thread_id,cpu,code,kctx,"), 0);
+
+ kbasep_ktrace_backend_format_header(buffer, sz, &written);
+
+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
+ ",info_val,ktrace_version=%u.%u",
+ KBASE_KTRACE_VERSION_MAJOR,
+ KBASE_KTRACE_VERSION_MINOR), 0);
+
+ buffer[sz - 1] = 0;
+}
+
+static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg,
+ char *buffer, int sz)
+{
+ s32 written = 0;
+
+ /* Initial part of message:
+ *
+ * secs,thread_id,cpu,code,kctx,
+ */
+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
+ "%d.%.6d,%d,%d,%s,%p,",
+ (int)trace_msg->timestamp.tv_sec,
+ (int)(trace_msg->timestamp.tv_nsec / 1000),
+ trace_msg->thread_id, trace_msg->cpu,
+ kbasep_ktrace_code_string[trace_msg->backend.code],
+ trace_msg->kctx), 0);
+
+ /* Backend parts */
+ kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz,
+ &written);
+
+ /* Rest of message:
+ *
+ * ,info_val
+ *
+ * Note that the last column is empty, it's simply to hold the ktrace
+ * version in the header
+ */
+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
+ ",0x%.16llx",
+ (unsigned long long)trace_msg->info_val), 0);
+ buffer[sz - 1] = 0;
+}
+
+static void kbasep_ktrace_dump_msg(struct kbase_device *kbdev,
+ struct kbase_ktrace_msg *trace_msg)
+{
+ char buffer[KTRACE_DUMP_MESSAGE_SIZE];
+
+ lockdep_assert_held(&kbdev->ktrace.lock);
+
+ kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer));
+ dev_dbg(kbdev->dev, "%s", buffer);
+}
+
+struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace)
+{
+ struct kbase_ktrace_msg *trace_msg;
+
+ lockdep_assert_held(&ktrace->lock);
+
+ trace_msg = &ktrace->rbuf[ktrace->next_in];
+
+ /* Update the ringbuffer indices */
+ ktrace->next_in = (ktrace->next_in + 1) & KBASE_KTRACE_MASK;
+ if (ktrace->next_in == ktrace->first_out)
+ ktrace->first_out = (ktrace->first_out + 1) & KBASE_KTRACE_MASK;
+
+ return trace_msg;
+}
+void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace,
+ struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code,
+ struct kbase_context *kctx, kbase_ktrace_flag_t flags,
+ u64 info_val)
+{
+ lockdep_assert_held(&ktrace->lock);
+
+ trace_msg->thread_id = task_pid_nr(current);
+ trace_msg->cpu = task_cpu(current);
+
+ ktime_get_real_ts64(&trace_msg->timestamp);
+
+ trace_msg->kctx = kctx;
+
+ trace_msg->info_val = info_val;
+ trace_msg->backend.code = code;
+ trace_msg->backend.flags = flags;
+}
+
+void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code,
+ struct kbase_context *kctx, kbase_ktrace_flag_t flags,
+ u64 info_val)
+{
+ unsigned long irqflags;
+ struct kbase_ktrace_msg *trace_msg;
+
+ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags);
+
+ /* Reserve and update indices */
+ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace);
+
+ /* Fill the common part of the message (including backend.flags) */
+ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags,
+ info_val);
+
+ /* Done */
+ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags);
+}
+
+static void kbasep_ktrace_clear_locked(struct kbase_device *kbdev)
+{
+ lockdep_assert_held(&kbdev->ktrace.lock);
+ kbdev->ktrace.first_out = kbdev->ktrace.next_in;
+}
+void kbasep_ktrace_clear(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->ktrace.lock, flags);
+ kbasep_ktrace_clear_locked(kbdev);
+ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
+}
+
+void kbasep_ktrace_dump(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ u32 start;
+ u32 end;
+ char buffer[KTRACE_DUMP_MESSAGE_SIZE] = "Dumping trace:\n";
+
+ kbasep_ktrace_format_header(buffer, sizeof(buffer), strlen(buffer));
+ dev_dbg(kbdev->dev, "%s", buffer);
+
+ spin_lock_irqsave(&kbdev->ktrace.lock, flags);
+ start = kbdev->ktrace.first_out;
+ end = kbdev->ktrace.next_in;
+
+ while (start != end) {
+ struct kbase_ktrace_msg *trace_msg = &kbdev->ktrace.rbuf[start];
+
+ kbasep_ktrace_dump_msg(kbdev, trace_msg);
+
+ start = (start + 1) & KBASE_KTRACE_MASK;
+ }
+ dev_dbg(kbdev->dev, "TRACE_END");
+
+ kbasep_ktrace_clear_locked(kbdev);
+
+ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
+}
+
+#ifdef CONFIG_DEBUG_FS
+struct trace_seq_state {
+ struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE];
+ u32 start;
+ u32 end;
+};
+
+static void *kbasep_ktrace_seq_start(struct seq_file *s, loff_t *pos)
+{
+ struct trace_seq_state *state = s->private;
+ int i;
+
+ if (*pos == 0)
+ /* See Documentation/filesystems/seq_file.txt */
+ return SEQ_START_TOKEN;
+
+ if (*pos > KBASE_KTRACE_SIZE)
+ return NULL;
+ i = state->start + *pos;
+ if ((state->end >= state->start && i >= state->end) ||
+ i >= state->end + KBASE_KTRACE_SIZE)
+ return NULL;
+
+ i &= KBASE_KTRACE_MASK;
+
+ return &state->trace_buf[i];
+}
+
+static void kbasep_ktrace_seq_stop(struct seq_file *s, void *data)
+{
+}
+
+static void *kbasep_ktrace_seq_next(struct seq_file *s, void *data, loff_t *pos)
+{
+ struct trace_seq_state *state = s->private;
+ int i;
+
+ if (data != SEQ_START_TOKEN)
+ (*pos)++;
+
+ i = (state->start + *pos) & KBASE_KTRACE_MASK;
+ if (i == state->end)
+ return NULL;
+
+ return &state->trace_buf[i];
+}
+
+static int kbasep_ktrace_seq_show(struct seq_file *s, void *data)
+{
+ struct kbase_ktrace_msg *trace_msg = data;
+ char buffer[KTRACE_DUMP_MESSAGE_SIZE];
+
+ /* If this is the start, print a header */
+ if (data == SEQ_START_TOKEN)
+ kbasep_ktrace_format_header(buffer, sizeof(buffer), 0);
+ else
+ kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer));
+
+ seq_printf(s, "%s\n", buffer);
+ return 0;
+}
+
+static const struct seq_operations kbasep_ktrace_seq_ops = {
+ .start = kbasep_ktrace_seq_start,
+ .next = kbasep_ktrace_seq_next,
+ .stop = kbasep_ktrace_seq_stop,
+ .show = kbasep_ktrace_seq_show,
+};
+
+static int kbasep_ktrace_debugfs_open(struct inode *inode, struct file *file)
+{
+ struct kbase_device *kbdev = inode->i_private;
+ unsigned long flags;
+
+ struct trace_seq_state *state;
+
+ state = __seq_open_private(file, &kbasep_ktrace_seq_ops,
+ sizeof(*state));
+ if (!state)
+ return -ENOMEM;
+
+ spin_lock_irqsave(&kbdev->ktrace.lock, flags);
+ state->start = kbdev->ktrace.first_out;
+ state->end = kbdev->ktrace.next_in;
+ memcpy(state->trace_buf, kbdev->ktrace.rbuf, sizeof(state->trace_buf));
+ spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
+
+ return 0;
+}
+
+static const struct file_operations kbasep_ktrace_debugfs_fops = {
+ .owner = THIS_MODULE,
+ .open = kbasep_ktrace_debugfs_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release_private,
+};
+
+void kbase_ktrace_debugfs_init(struct kbase_device *kbdev)
+{
+ debugfs_create_file("mali_trace", 0444,
+ kbdev->mali_debugfs_directory, kbdev,
+ &kbasep_ktrace_debugfs_fops);
+}
+#endif /* CONFIG_DEBUG_FS */
+
+#else /* KBASE_KTRACE_TARGET_RBUF */
+
+#ifdef CONFIG_DEBUG_FS
+void kbase_ktrace_debugfs_init(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+#endif /* CONFIG_DEBUG_FS */
+#endif /* KBASE_KTRACE_TARGET_RBUF */
diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace.h b/mali_kbase/debug/mali_kbase_debug_ktrace.h
new file mode 100644
index 0000000..0dd8b7a
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_ktrace.h
@@ -0,0 +1,219 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/*
+ * DOC: Kbase's own trace, 'KTrace'
+ *
+ * Low overhead trace specific to kbase, aimed at:
+ * - common use-cases for tracing kbase specific functionality to do with
+ * running work on the GPU
+ * - easy 1-line addition of new types of trace
+ *
+ * KTrace can be recorded in one or more of the following targets:
+ * - KBASE_KTRACE_TARGET_RBUF: low overhead ringbuffer protected by an
+ * irq-spinlock, output available via dev_dbg() and debugfs file
+ * - KBASE_KTRACE_TARGET_FTRACE: ftrace based tracepoints under 'mali' events
+ */
+
+#ifndef _KBASE_DEBUG_KTRACE_H_
+#define _KBASE_DEBUG_KTRACE_H_
+
+#include "debug/backend/mali_kbase_debug_ktrace_jm.h"
+
+/**
+ * kbase_ktrace_init - initialize kbase ktrace.
+ * @kbdev: kbase device
+ */
+int kbase_ktrace_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_ktrace_term - terminate kbase ktrace.
+ * @kbdev: kbase device
+ */
+void kbase_ktrace_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_ktrace_hook_wrapper - wrapper so that dumping ktrace can be done via a
+ * callback.
+ * @param: kbase device, cast to void pointer
+ */
+void kbase_ktrace_hook_wrapper(void *param);
+
+#ifdef CONFIG_DEBUG_FS
+/**
+ * kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if
+ * the selected targets support it.
+ * @kbdev: kbase device
+ *
+ * There is no matching 'term' call, debugfs_remove_recursive() is sufficient.
+ */
+void kbase_ktrace_debugfs_init(struct kbase_device *kbdev);
+#endif /* CONFIG_DEBUG_FS */
+
+/*
+ * KTrace target for internal ringbuffer
+ */
+#if KBASE_KTRACE_TARGET_RBUF
+/**
+ * kbasep_ktrace_add - internal function to add trace to the ringbuffer.
+ * @kbdev: kbase device
+ * @code: ktrace code
+ * @kctx: kbase context, or NULL if no context
+ * @flags: flags about the message
+ * @info_val: generic information about @code to add to the trace
+ *
+ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD() instead.
+ */
+void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code,
+ struct kbase_context *kctx, kbase_ktrace_flag_t flags,
+ u64 info_val);
+
+/**
+ * kbasep_ktrace_clear - clear the trace ringbuffer
+ * @kbdev: kbase device
+ *
+ * PRIVATE: do not use directly. Use KBASE_KTRACE_CLEAR() instead.
+ */
+void kbasep_ktrace_clear(struct kbase_device *kbdev);
+
+/**
+ * kbasep_ktrace_dump - dump ktrace ringbuffer to dev_dbg(), then clear it
+ * @kbdev: kbase device
+ *
+ * PRIVATE: do not use directly. Use KBASE_KTRACE_DUMP() instead.
+ */
+void kbasep_ktrace_dump(struct kbase_device *kbdev);
+
+#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \
+ kbasep_ktrace_add(kbdev, KBASE_KTRACE_CODE(code), kctx, 0, \
+ info_val) \
+
+#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \
+ kbasep_ktrace_clear(kbdev)
+
+#define KBASE_KTRACE_RBUF_DUMP(kbdev) \
+ kbasep_ktrace_dump(kbdev)
+
+#else /* KBASE_KTRACE_TARGET_RBUF */
+
+#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(code); \
+ CSTD_UNUSED(kctx); \
+ CSTD_UNUSED(info_val); \
+ CSTD_NOP(0); \
+ } while (0)
+
+#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(0); \
+ } while (0)
+#define KBASE_KTRACE_RBUF_DUMP(kbdev) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(0); \
+ } while (0)
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+
+/*
+ * KTrace target for Linux's ftrace
+ */
+#if KBASE_KTRACE_TARGET_FTRACE
+#include "mali_linux_trace.h"
+
+#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \
+ trace_mali_##code(info_val)
+
+#else /* KBASE_KTRACE_TARGET_FTRACE */
+#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(code); \
+ CSTD_UNUSED(kctx); \
+ CSTD_UNUSED(info_val); \
+ CSTD_NOP(0); \
+ } while (0)
+#endif /* KBASE_KTRACE_TARGET_FTRACE */
+
+/* No 'clear' implementation for ftrace yet */
+#define KBASE_KTRACE_FTRACE_CLEAR(kbdev) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(0); \
+ } while (0)
+
+/* No 'dump' implementation for ftrace yet */
+#define KBASE_KTRACE_FTRACE_DUMP(kbdev) \
+ do { \
+ CSTD_UNUSED(kbdev); \
+ CSTD_NOP(0); \
+ } while (0)
+
+/*
+ * Master set of macros to route KTrace to any of the targets
+ */
+
+/**
+ * KBASE_KTRACE_ADD - Add trace values
+ * @kbdev: kbase device
+ * @code: trace code
+ * @kctx: kbase context, or NULL if no context
+ * @info_val: generic information about @code to add to the trace
+ *
+ * Note: Any functions called through this macro will still be evaluated in
+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when
+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
+ * to this macro must:
+ * a) be static or static inline, and
+ * b) just return 0 and have no other statements present in the body.
+ */
+#define KBASE_KTRACE_ADD(kbdev, code, kctx, info_val) \
+ do { \
+ /* capture values that could come from non-pure function calls */ \
+ u64 __info_val = info_val; \
+ KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, __info_val); \
+ KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, __info_val); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_CLEAR - Clear the trace, if applicable to the target(s)
+ * @kbdev: kbase device
+ */
+#define KBASE_KTRACE_CLEAR(kbdev) \
+ do { \
+ KBASE_KTRACE_RBUF_CLEAR(kbdev); \
+ KBASE_KTRACE_FTRACE_CLEAR(kbdev); \
+ } while (0)
+
+/**
+ * KBASE_KTRACE_DUMP - Dump the trace, if applicable to the target(s)
+ * @kbdev: kbase device
+ */
+#define KBASE_KTRACE_DUMP(kbdev) \
+ do { \
+ KBASE_KTRACE_RBUF_DUMP(kbdev); \
+ KBASE_KTRACE_FTRACE_DUMP(kbdev); \
+ } while (0)
+
+#endif /* _KBASE_DEBUG_KTRACE_H_ */
diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h
new file mode 100644
index 0000000..364ed60
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_ktrace_codes.h
@@ -0,0 +1,158 @@
+/*
+ *
+ * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/*
+ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE *****
+ * ***** DO NOT INCLUDE DIRECTLY *****
+ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL *****
+ */
+
+/*
+ * The purpose of this header file is just to contain a list of trace code
+ * identifiers
+ *
+ * Each identifier is wrapped in a macro, so that its string form and enum form
+ * can be created
+ *
+ * Each macro is separated with a comma, to allow insertion into an array
+ * initializer or enum definition block.
+ *
+ * This allows automatic creation of an enum and a corresponding array of
+ * strings
+ *
+ * Before #including, the includer MUST #define KBASE_KTRACE_CODE_MAKE_CODE.
+ * After #including, the includer MUST #under KBASE_KTRACE_CODE_MAKE_CODE.
+ *
+ * e.g.:
+ * #define KBASE_KTRACE_CODE( X ) KBASE_KTRACE_CODE_ ## X
+ * typedef enum
+ * {
+ * #define KBASE_KTRACE_CODE_MAKE_CODE( X ) KBASE_KTRACE_CODE( X )
+ * #include "mali_kbase_debug_ktrace_codes.h"
+ * #undef KBASE_KTRACE_CODE_MAKE_CODE
+ * } kbase_ktrace_code;
+ *
+ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE
+ *
+ *
+ * The use of the macro here is:
+ * - KBASE_KTRACE_CODE_MAKE_CODE( X )
+ *
+ * Which produces:
+ * - For an enum, KBASE_KTRACE_CODE_X
+ * - For a string, "X"
+ *
+ *
+ * For example:
+ * - KBASE_KTRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to:
+ * - KBASE_KTRACE_CODE_JM_JOB_COMPLETE for the enum
+ * - "JM_JOB_COMPLETE" for the string
+ * - To use it to trace an event, do:
+ * - KBASE_KTRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val );
+ */
+
+#if 0 /* Dummy section to avoid breaking formatting */
+int dummy_array[] = {
+#endif
+
+ /*
+ * Core events
+ */
+ /* no info_val */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY),
+ /* no info_val */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM),
+ /* info_val == GPU_IRQ_STATUS register */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ),
+ /* info_val == bits cleared */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR),
+ /* info_val == GPU_IRQ_STATUS register */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE),
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET),
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET),
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR),
+ /* info_val == dump address */
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE),
+ KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES),
+
+ /*
+ * Power Management Events
+ */
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_L2),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_L2),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS),
+ /* info_val == kbdev->pm.active_count*/
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE),
+ /* info_val == kbdev->pm.active_count*/
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_ON),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_OFF),
+ /* info_val == policy number, or -1 for "Already changing" */
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_SET_POLICY),
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY),
+ /* info_val == policy number */
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT),
+ /* info_val == policy number */
+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM),
+
+ /*
+ * Context Scheduler events
+ */
+ /* info_val == kctx->refcount */
+ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK),
+ /* info_val == kctx->refcount */
+ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX),
+
+
+#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h"
+ /*
+ * Unused code just to make it easier to not have a comma at the end.
+ * All other codes MUST come before this
+ */
+ KBASE_KTRACE_CODE_MAKE_CODE(DUMMY)
+
+#if 0 /* Dummy section to avoid breaking formatting */
+};
+#endif
+
+/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */
diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h
new file mode 100644
index 0000000..d6baaf1
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_ktrace_defs.h
@@ -0,0 +1,152 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+#ifndef _KBASE_DEBUG_KTRACE_DEFS_H_
+#define _KBASE_DEBUG_KTRACE_DEFS_H_
+
+/* Enable SW tracing when set */
+#if defined(CONFIG_MALI_MIDGARD_ENABLE_TRACE) || defined(CONFIG_MALI_SYSTEM_TRACE)
+#define KBASE_KTRACE_ENABLE 1
+#endif
+
+#ifndef KBASE_KTRACE_ENABLE
+#ifdef CONFIG_MALI_DEBUG
+#define KBASE_KTRACE_ENABLE 1
+#else /* CONFIG_MALI_DEBUG */
+#define KBASE_KTRACE_ENABLE 0
+#endif /* CONFIG_MALI_DEBUG */
+#endif /* KBASE_KTRACE_ENABLE */
+
+/* Select targets for recording of trace:
+ *
+ */
+#if KBASE_KTRACE_ENABLE
+
+#ifdef CONFIG_MALI_SYSTEM_TRACE
+#define KBASE_KTRACE_TARGET_FTRACE 1
+#else /* CONFIG_MALI_SYSTEM_TRACE */
+#define KBASE_KTRACE_TARGET_FTRACE 0
+#endif /* CONFIG_MALI_SYSTEM_TRACE */
+
+#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE
+#define KBASE_KTRACE_TARGET_RBUF 1
+#else /* CONFIG_MALI_MIDGARD_ENABLE_TRACE*/
+#define KBASE_KTRACE_TARGET_RBUF 0
+#endif /* CONFIG_MALI_MIDGARD_ENABLE_TRACE */
+
+#else /* KBASE_KTRACE_ENABLE */
+#define KBASE_KTRACE_TARGET_FTRACE 0
+#define KBASE_KTRACE_TARGET_RBUF 0
+#endif /* KBASE_KTRACE_ENABLE */
+
+/*
+ * NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in
+ * the backend, since updates can be made to one backend in a way that doesn't
+ * affect the other.
+ *
+ * However, modifying the common part could require both backend versions to be
+ * updated.
+ */
+
+#if KBASE_KTRACE_TARGET_RBUF
+typedef u8 kbase_ktrace_flag_t;
+typedef u8 kbase_ktrace_code_t;
+
+/*
+ * struct kbase_ktrace_backend - backend specific part of a trace message
+ *
+ * At the very least, this must contain a kbase_ktrace_code_t 'code' member and
+ * a kbase_ktrace_flag_t 'flags' member
+ */
+struct kbase_ktrace_backend;
+
+#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h"
+
+/* Indicates if the trace message has backend related info.
+ *
+ * If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg
+ * as uninitialized, apart from the mandatory parts:
+ * - code
+ * - flags
+ */
+#define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7)
+
+#define KBASE_KTRACE_SHIFT 8 /* 256 entries */
+#define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT)
+#define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1)
+
+#define KBASE_KTRACE_CODE(X) KBASE_KTRACE_CODE_ ## X
+
+/* Note: compiletime_assert() about this against kbase_ktrace_code_t is in
+ * kbase_ktrace_init()
+ */
+enum kbase_ktrace_code {
+ /*
+ * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
+ * THIS MUST BE USED AT THE START OF THE ENUM
+ */
+#define KBASE_KTRACE_CODE_MAKE_CODE(X) KBASE_KTRACE_CODE(X)
+#include <debug/mali_kbase_debug_ktrace_codes.h>
+#undef KBASE_KTRACE_CODE_MAKE_CODE
+ /* Comma on its own, to extend the list */
+ ,
+ /* Must be the last in the enum */
+ KBASE_KTRACE_CODE_COUNT
+};
+
+/**
+ * struct kbase_ktrace - object representing a trace message added to trace
+ * buffer trace_rbuf in &kbase_device
+ * @timestamp: CPU timestamp at which the trace message was added.
+ * @thread_id: id of the thread in the context of which trace message was
+ * added.
+ * @cpu: indicates which CPU the @thread_id was scheduled on when the
+ * trace message was added.
+ * @kctx: Pointer to the kbase context for which the trace message was
+ * added. Will be NULL for certain trace messages associated with
+ * the &kbase_device itself, such as power management events.
+ * Will point to the appropriate context corresponding to
+ * backend-specific events.
+ * @info_val: value specific to the type of event being traced. Refer to the
+ * specific code in enum kbase_ktrace_code
+ * @backend: backend-specific trace information. All backends must implement
+ * a minimum common set of members
+ */
+struct kbase_ktrace_msg {
+ struct timespec64 timestamp;
+ u32 thread_id;
+ u32 cpu;
+ void *kctx;
+ u64 info_val;
+
+ struct kbase_ktrace_backend backend;
+};
+
+struct kbase_ktrace {
+ spinlock_t lock;
+ u16 first_out;
+ u16 next_in;
+ struct kbase_ktrace_msg *rbuf;
+};
+
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+#endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */
diff --git a/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h
new file mode 100644
index 0000000..e450760
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_ktrace_internal.h
@@ -0,0 +1,89 @@
+/*
+ *
+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+#ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_
+#define _KBASE_DEBUG_KTRACE_INTERNAL_H_
+
+#if KBASE_KTRACE_TARGET_RBUF
+
+#define KTRACE_DUMP_MESSAGE_SIZE 256
+
+/**
+ * kbasep_ktrace_backend_format_header - format the backend part of the header
+ * @buffer: buffer to write to
+ * @sz: size of @buffer in bytes
+ * @written: pointer to storage for updating bytes written so far to @buffer
+ *
+ * The backend must format only the non-common backend specific parts of the
+ * header. It must format them as though they were standalone. The caller will
+ * handle adding any delimiters around this.
+ */
+void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written);
+
+/**
+ * kbasep_ktrace_backend_format_msg - format the backend part of the message
+ * @trace_msg: ktrace message
+ * @buffer: buffer to write to
+ * @sz: size of @buffer in bytes
+ * @written: pointer to storage for updating bytes written so far to @buffer
+ *
+ * The backend must format only the non-common backend specific parts of the
+ * message. It must format them as though they were standalone. The caller will
+ * handle adding any delimiters around this.
+ *
+ * A caller may have the flags member of @trace_msg with
+ * %KBASE_KTRACE_FLAG_BACKEND clear. The backend must handle that setting
+ * appropriately.
+ */
+void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg,
+ char *buffer, int sz, s32 *written);
+
+
+/**
+ * kbasep_ktrace_reserve - internal function to reserve space for a ktrace
+ * message
+ * @ktrace: kbase device's ktrace
+ *
+ * This may also empty the oldest entry in the ringbuffer to make space.
+ */
+struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace);
+
+/**
+ * kbasep_ktrace_msg_init - internal function to initialize just the common
+ * part of a ktrace message
+ * @ktrace: kbase device's ktrace
+ * @trace_msg: ktrace message to initialize
+ * @code: ktrace code
+ * @kctx: kbase context, or NULL if no context
+ * @flags: flags about the message
+ * @info_val: generic information about @code to add to the trace
+ *
+ * The common part includes the mandatory parts of the backend part
+ */
+void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace,
+ struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code,
+ struct kbase_context *kctx, kbase_ktrace_flag_t flags,
+ u64 info_val);
+
+#endif /* KBASE_KTRACE_TARGET_RBUF */
+
+#endif /* _KBASE_DEBUG_KTRACE_INTERNAL_H_ */
diff --git a/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h
new file mode 100644
index 0000000..18e4f7c
--- /dev/null
+++ b/mali_kbase/debug/mali_kbase_debug_linux_ktrace.h
@@ -0,0 +1,99 @@
+/*
+ *
+ * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ */
+
+/*
+ * NOTE: This must **only** be included through mali_linux_trace.h,
+ * otherwise it will fail to setup tracepoints correctly
+ */
+
+#if !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _KBASE_DEBUG_LINUX_KTRACE_H_
+
+#if KBASE_KTRACE_TARGET_FTRACE
+
+DECLARE_EVENT_CLASS(mali_add_template,
+ TP_PROTO(u64 info_val),
+ TP_ARGS(info_val),
+ TP_STRUCT__entry(
+ __field(u64, info_val)
+ ),
+ TP_fast_assign(
+ __entry->info_val = info_val;
+ ),
+ TP_printk("info=0x%llx", __entry->info_val)
+);
+
+#define DEFINE_MALI_ADD_EVENT(name) \
+DEFINE_EVENT(mali_add_template, mali_##name, \
+ TP_PROTO(u64 info_val), \
+ TP_ARGS(info_val))
+DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY);
+DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR);
+DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES);
+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED);
+DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP);
+DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP);
+DEFINE_MALI_ADD_EVENT(PM_PWRON);
+DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER);
+DEFINE_MALI_ADD_EVENT(PM_PWRON_L2);
+DEFINE_MALI_ADD_EVENT(PM_PWROFF);
+DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER);
+DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2);
+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED);
+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER);
+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2);
+DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED);
+DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER);
+DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED);
+DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED);
+DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED);
+DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED);
+DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE);
+DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER);
+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE);
+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER);
+DEFINE_MALI_ADD_EVENT(PM_GPU_ON);
+DEFINE_MALI_ADD_EVENT(PM_GPU_OFF);
+DEFINE_MALI_ADD_EVENT(PM_SET_POLICY);
+DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT);
+DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM);
+DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY);
+DEFINE_MALI_ADD_EVENT(PM_CONTEXT_ACTIVE);
+DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE);
+DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS);
+DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK);
+DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX);
+
+#undef DEFINE_MALI_ADD_EVENT
+
+#include "mali_kbase_debug_linux_ktrace_jm.h"
+
+#endif /* KBASE_KTRACE_TARGET_FTRACE */
+
+#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) */
diff --git a/mali_kbase/device/backend/mali_kbase_device_jm.c b/mali_kbase/device/backend/mali_kbase_device_jm.c
index 24dbe80..fbba2e7 100644
--- a/mali_kbase/device/backend/mali_kbase_device_jm.c
+++ b/mali_kbase/device/backend/mali_kbase_device_jm.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
@@ -32,6 +33,10 @@
#include <mali_kbase_model_linux.h>
#endif
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include <arbiter/mali_kbase_arbiter_pm.h>
+#endif
+
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
@@ -94,12 +99,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev)
kbase_pm_context_idle(kbdev);
/* Update gpuprops with L2_FEATURES if applicable */
- kbase_gpuprops_update_l2_features(kbdev);
+ err = kbase_gpuprops_update_l2_features(kbdev);
+ if (err)
+ goto fail_update_l2_features;
init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);
return 0;
+fail_update_l2_features:
fail_devfreq_init:
kbase_job_slot_term(kbdev);
fail_job_slot:
@@ -146,10 +154,10 @@ static const struct kbase_device_init dev_init[] = {
{registers_map, registers_unmap,
"Register map failed"},
#endif
- {power_control_init, power_control_term,
- "Power control initialization failed"},
{kbase_device_io_history_init, kbase_device_io_history_term,
"Register access history initialization failed"},
+ {kbase_device_pm_init, kbase_device_pm_term,
+ "Power management initialization failed"},
{kbase_device_early_init, kbase_device_early_term,
"Early device initialization failed"},
{kbase_device_populate_max_freq, NULL,
diff --git a/mali_kbase/device/mali_kbase_device.c b/mali_kbase/device/mali_kbase_device.c
index 4c77929..76f14e5 100644
--- a/mali_kbase/device/mali_kbase_device.c
+++ b/mali_kbase/device/mali_kbase_device.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
@@ -50,24 +51,16 @@
#include "backend/gpu/mali_kbase_pm_internal.h"
#include "backend/gpu/mali_kbase_irq_internal.h"
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include "arbiter/mali_kbase_arbiter_pm.h"
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+
/* NOTE: Magic - 0x45435254 (TRCE in ASCII).
* Supports tracing feature provided in the base module.
* Please keep it in sync with the value of base module.
*/
#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254
-#if KBASE_TRACE_ENABLE
-static const char *kbasep_trace_code_string[] = {
- /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
- * THIS MUST BE USED AT THE START OF THE ARRAY */
-#define KBASE_TRACE_CODE_MAKE_CODE(X) # X
-#include "tl/mali_kbase_trace_defs.h"
-#undef KBASE_TRACE_CODE_MAKE_CODE
-};
-#endif
-
-#define DEBUG_MESSAGE_SIZE 256
-
/* Number of register accesses for the buffer that we allocate during
* initialization time. The buffer size can be changed later via debugfs.
*/
@@ -77,10 +70,6 @@ static DEFINE_MUTEX(kbase_dev_list_lock);
static LIST_HEAD(kbase_dev_list);
static int kbase_dev_nr;
-static int kbasep_trace_init(struct kbase_device *kbdev);
-static void kbasep_trace_term(struct kbase_device *kbdev);
-static void kbasep_trace_hook_wrapper(void *param);
-
struct kbase_device *kbase_device_alloc(void)
{
return kzalloc(sizeof(struct kbase_device), GFP_KERNEL);
@@ -176,7 +165,9 @@ int kbase_device_misc_init(struct kbase_device * const kbdev)
*/
kbase_hw_set_features_mask(kbdev);
- kbase_gpuprops_set_features(kbdev);
+ err = kbase_gpuprops_set_features(kbdev);
+ if (err)
+ goto fail;
/* On Linux 4.0+, dma coherency is determined from device tree */
#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
@@ -207,13 +198,13 @@ int kbase_device_misc_init(struct kbase_device * const kbdev)
spin_lock_init(&kbdev->hwcnt.lock);
- err = kbasep_trace_init(kbdev);
+ err = kbase_ktrace_init(kbdev);
if (err)
goto term_as;
init_waitqueue_head(&kbdev->cache_clean_wait);
- kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev);
+ kbase_debug_assert_register_hook(&kbase_ktrace_hook_wrapper, kbdev);
atomic_set(&kbdev->ctx_num, 0);
@@ -233,9 +224,11 @@ int kbase_device_misc_init(struct kbase_device * const kbdev)
mutex_init(&kbdev->kctx_list_lock);
INIT_LIST_HEAD(&kbdev->kctx_list);
+ spin_lock_init(&kbdev->hwaccess_lock);
+
return 0;
term_trace:
- kbasep_trace_term(kbdev);
+ kbase_ktrace_term(kbdev);
term_as:
kbase_device_all_as_term(kbdev);
as_init_failed:
@@ -250,13 +243,13 @@ void kbase_device_misc_term(struct kbase_device *kbdev)
WARN_ON(!list_empty(&kbdev->kctx_list));
-#if KBASE_TRACE_ENABLE
+#if KBASE_KTRACE_ENABLE
kbase_debug_assert_register_hook(NULL, NULL);
#endif
kbase_instr_backend_term(kbdev);
- kbasep_trace_term(kbdev);
+ kbase_ktrace_term(kbdev);
kbase_device_all_as_term(kbdev);
}
@@ -313,9 +306,8 @@ void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev)
int kbase_device_timeline_init(struct kbase_device *kbdev)
{
- atomic_set(&kbdev->timeline_is_enabled, 0);
- return kbase_timeline_init(&kbdev->timeline,
- &kbdev->timeline_is_enabled);
+ atomic_set(&kbdev->timeline_flags, 0);
+ return kbase_timeline_init(&kbdev->timeline, &kbdev->timeline_flags);
}
void kbase_device_timeline_term(struct kbase_device *kbdev)
@@ -387,274 +379,6 @@ void kbase_device_put_list(const struct list_head *dev_list)
}
KBASE_EXPORT_TEST_API(kbase_device_put_list);
-/*
- * Device trace functions
- */
-#if KBASE_TRACE_ENABLE
-
-static int kbasep_trace_init(struct kbase_device *kbdev)
-{
- struct kbase_trace *rbuf;
-
- rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL);
-
- if (!rbuf)
- return -EINVAL;
-
- kbdev->trace_rbuf = rbuf;
- spin_lock_init(&kbdev->trace_lock);
- return 0;
-}
-
-static void kbasep_trace_term(struct kbase_device *kbdev)
-{
- kfree(kbdev->trace_rbuf);
-}
-
-static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len)
-{
- s32 written = 0;
-
- /* Initial part of message */
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0);
-
- if (trace_msg->katom)
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0);
-
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0);
-
- /* NOTE: Could add function callbacks to handle different message types */
- /* Jobslot present */
- if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT)
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0);
-
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
-
- /* Refcount present */
- if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT)
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0);
-
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
-
- /* Rest of message */
- written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0);
-}
-
-static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg)
-{
- char buffer[DEBUG_MESSAGE_SIZE];
-
- kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE);
- dev_dbg(kbdev->dev, "%s", buffer);
-}
-
-void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val)
-{
- unsigned long irqflags;
- struct kbase_trace *trace_msg;
-
- spin_lock_irqsave(&kbdev->trace_lock, irqflags);
-
- trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in];
-
- /* Fill the message */
- trace_msg->thread_id = task_pid_nr(current);
- trace_msg->cpu = task_cpu(current);
-
- getnstimeofday(&trace_msg->timestamp);
-
- trace_msg->code = code;
- trace_msg->ctx = ctx;
-
- if (NULL == katom) {
- trace_msg->katom = false;
- } else {
- trace_msg->katom = true;
- trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom);
- trace_msg->atom_udata[0] = katom->udata.blob[0];
- trace_msg->atom_udata[1] = katom->udata.blob[1];
- }
-
- trace_msg->gpu_addr = gpu_addr;
- trace_msg->jobslot = jobslot;
- trace_msg->refcount = MIN((unsigned int)refcount, 0xFF);
- trace_msg->info_val = info_val;
- trace_msg->flags = flags;
-
- /* Update the ringbuffer indices */
- kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK;
- if (kbdev->trace_next_in == kbdev->trace_first_out)
- kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK;
-
- /* Done */
-
- spin_unlock_irqrestore(&kbdev->trace_lock, irqflags);
-}
-
-void kbasep_trace_clear(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->trace_lock, flags);
- kbdev->trace_first_out = kbdev->trace_next_in;
- spin_unlock_irqrestore(&kbdev->trace_lock, flags);
-}
-
-void kbasep_trace_dump(struct kbase_device *kbdev)
-{
- unsigned long flags;
- u32 start;
- u32 end;
-
- dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val");
- spin_lock_irqsave(&kbdev->trace_lock, flags);
- start = kbdev->trace_first_out;
- end = kbdev->trace_next_in;
-
- while (start != end) {
- struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start];
-
- kbasep_trace_dump_msg(kbdev, trace_msg);
-
- start = (start + 1) & KBASE_TRACE_MASK;
- }
- dev_dbg(kbdev->dev, "TRACE_END");
-
- spin_unlock_irqrestore(&kbdev->trace_lock, flags);
-
- KBASE_TRACE_CLEAR(kbdev);
-}
-
-static void kbasep_trace_hook_wrapper(void *param)
-{
- struct kbase_device *kbdev = (struct kbase_device *)param;
-
- kbasep_trace_dump(kbdev);
-}
-
-#ifdef CONFIG_DEBUG_FS
-struct trace_seq_state {
- struct kbase_trace trace_buf[KBASE_TRACE_SIZE];
- u32 start;
- u32 end;
-};
-
-static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos)
-{
- struct trace_seq_state *state = s->private;
- int i;
-
- if (*pos > KBASE_TRACE_SIZE)
- return NULL;
- i = state->start + *pos;
- if ((state->end >= state->start && i >= state->end) ||
- i >= state->end + KBASE_TRACE_SIZE)
- return NULL;
-
- i &= KBASE_TRACE_MASK;
-
- return &state->trace_buf[i];
-}
-
-static void kbasep_trace_seq_stop(struct seq_file *s, void *data)
-{
-}
-
-static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos)
-{
- struct trace_seq_state *state = s->private;
- int i;
-
- (*pos)++;
-
- i = (state->start + *pos) & KBASE_TRACE_MASK;
- if (i == state->end)
- return NULL;
-
- return &state->trace_buf[i];
-}
-
-static int kbasep_trace_seq_show(struct seq_file *s, void *data)
-{
- struct kbase_trace *trace_msg = data;
- char buffer[DEBUG_MESSAGE_SIZE];
-
- kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE);
- seq_printf(s, "%s\n", buffer);
- return 0;
-}
-
-static const struct seq_operations kbasep_trace_seq_ops = {
- .start = kbasep_trace_seq_start,
- .next = kbasep_trace_seq_next,
- .stop = kbasep_trace_seq_stop,
- .show = kbasep_trace_seq_show,
-};
-
-static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file)
-{
- struct kbase_device *kbdev = inode->i_private;
- unsigned long flags;
-
- struct trace_seq_state *state;
-
- state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state));
- if (!state)
- return -ENOMEM;
-
- spin_lock_irqsave(&kbdev->trace_lock, flags);
- state->start = kbdev->trace_first_out;
- state->end = kbdev->trace_next_in;
- memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf));
- spin_unlock_irqrestore(&kbdev->trace_lock, flags);
-
- return 0;
-}
-
-static const struct file_operations kbasep_trace_debugfs_fops = {
- .owner = THIS_MODULE,
- .open = kbasep_trace_debugfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release_private,
-};
-
-void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
-{
- debugfs_create_file("mali_trace", S_IRUGO,
- kbdev->mali_debugfs_directory, kbdev,
- &kbasep_trace_debugfs_fops);
-}
-
-#else
-void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
-{
-}
-#endif /* CONFIG_DEBUG_FS */
-
-#else /* KBASE_TRACE_ENABLE */
-static int kbasep_trace_init(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
- return 0;
-}
-
-static void kbasep_trace_term(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-static void kbasep_trace_hook_wrapper(void *param)
-{
- CSTD_UNUSED(param);
-}
-
-void kbasep_trace_dump(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-#endif /* KBASE_TRACE_ENABLE */
-
int kbase_device_early_init(struct kbase_device *kbdev)
{
int err;
@@ -692,7 +416,14 @@ fail_runtime_pm:
void kbase_device_early_term(struct kbase_device *kbdev)
{
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbdev->arb.arb_if)
+ kbase_arbiter_pm_release_interrupts(kbdev);
+ else
+ kbase_release_interrupts(kbdev);
+#else
kbase_release_interrupts(kbdev);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
kbase_pm_runtime_term(kbdev);
kbasep_platform_device_term(kbdev);
}
diff --git a/mali_kbase/gpu/mali_kbase_gpu_regmap.h b/mali_kbase/gpu/mali_kbase_gpu_regmap.h
index 31abae2..759f30d 100644
--- a/mali_kbase/gpu/mali_kbase_gpu_regmap.h
+++ b/mali_kbase/gpu/mali_kbase_gpu_regmap.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -232,9 +232,17 @@
#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */
#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */
-#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \
+/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test.
+ */
+#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \
| POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED)
+#ifdef CONFIG_MALI_DEBUG
+#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE)
+#else /* CONFIG_MALI_DEBUG */
+#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON)
+#endif /* CONFIG_MALI_DEBUG */
+
/*
* MMU_IRQ_RAWSTAT register values. Values are valid also for
* MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers.
diff --git a/mali_kbase/jm/mali_base_jm_kernel.h b/mali_kbase/jm/mali_base_jm_kernel.h
index b61e612..879a436 100644
--- a/mali_kbase/jm/mali_base_jm_kernel.h
+++ b/mali_kbase/jm/mali_base_jm_kernel.h
@@ -264,7 +264,6 @@ typedef u32 base_context_create_flags;
#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \
BASE_TLSTREAM_JOB_DUMPING_ENABLED)
-
/*
* Dependency stuff, keep it private for now. May want to expose it if
* we decide to make the number of semaphores a configurable
diff --git a/mali_kbase/jm/mali_kbase_jm_defs.h b/mali_kbase/jm/mali_kbase_jm_defs.h
index 172217f..aac561b 100644
--- a/mali_kbase/jm/mali_kbase_jm_defs.h
+++ b/mali_kbase/jm/mali_kbase_jm_defs.h
@@ -30,8 +30,10 @@
#ifndef _KBASE_JM_DEFS_H_
#define _KBASE_JM_DEFS_H_
-/* Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */
-#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1
+#include "mali_kbase_js_defs.h"
+
+/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */
+#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1
/*
* Number of milliseconds before resetting the GPU when a job cannot be "zapped"
@@ -770,6 +772,12 @@ struct kbase_jd_renderpass {
* reaching this offset.
* @work_id: atomic variable used for GPU tracepoints,
* incremented on every call to base_jd_submit.
+ * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
+ * allocate & free, in submission order, protected
+ * by kbase_jd_context.lock.
+ * @jit_pending_alloc: A list of just-in-time memory allocation
+ * soft-jobs which will be reattempted after the
+ * impending free of other active allocations.
*/
struct kbase_jd_context {
struct mutex lock;
@@ -787,6 +795,9 @@ struct kbase_jd_context {
#ifdef CONFIG_GPU_TRACEPOINTS
atomic_t work_id;
#endif
+
+ struct list_head jit_atoms_head;
+ struct list_head jit_pending_alloc;
};
/**
diff --git a/mali_kbase/jm/mali_kbase_jm_ioctl.h b/mali_kbase/jm/mali_kbase_jm_ioctl.h
index 127d990..408e98e 100644
--- a/mali_kbase/jm/mali_kbase_jm_ioctl.h
+++ b/mali_kbase/jm/mali_kbase_jm_ioctl.h
@@ -92,9 +92,11 @@
* flags member. Previous variants of this structure are kept and given _10_2
* and _11_5 suffixes.
* - The above changes are checked for safe values in usual builds
+ * 11.21:
+ * - v2.0 of mali_trace debugfs file, which now versions the file separately
*/
#define BASE_UK_VERSION_MAJOR 11
-#define BASE_UK_VERSION_MINOR 20
+#define BASE_UK_VERSION_MINOR 21
/**
* struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel
diff --git a/mali_kbase/mali_kbase_js_defs.h b/mali_kbase/jm/mali_kbase_js_defs.h
index f858687..0b48615 100644
--- a/mali_kbase/mali_kbase_js_defs.h
+++ b/mali_kbase/jm/mali_kbase_js_defs.h
@@ -21,7 +21,6 @@
*/
-
/**
* @file mali_kbase_js.h
* Job Scheduler Type Definitions
diff --git a/mali_kbase/mali_base_hwconfig_issues.h b/mali_kbase/mali_base_hwconfig_issues.h
index 94c89fa..3966069 100644
--- a/mali_kbase/mali_base_hwconfig_issues.h
+++ b/mali_kbase/mali_base_hwconfig_issues.h
@@ -58,6 +58,7 @@ enum base_hw_issue {
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@@ -349,6 +350,7 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = {
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@@ -364,6 +366,7 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = {
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@@ -406,6 +409,7 @@ static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = {
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@@ -446,6 +450,7 @@ static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = {
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@@ -500,6 +505,33 @@ static const enum base_hw_issue base_hw_issues_model_tBEx[] = {
BASE_HW_ISSUE_END
};
+static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = {
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TSIX_2033,
+ BASE_HW_ISSUE_TTRX_1337,
+ BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
+ BASE_HW_ISSUE_TTRX_921,
+ BASE_HW_ISSUE_TTRX_3414,
+ BASE_HW_ISSUE_TTRX_3083,
+ BASE_HW_ISSUE_TTRX_3470,
+ BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_TTRX_3485,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = {
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_TSIX_2033,
+ BASE_HW_ISSUE_TTRX_1337,
+ BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
+ BASE_HW_ISSUE_TTRX_921,
+ BASE_HW_ISSUE_TTRX_3414,
+ BASE_HW_ISSUE_TTRX_3083,
+ BASE_HW_ISSUE_TTRX_3470,
+ BASE_HW_ISSUE_TTRX_3464,
+ BASE_HW_ISSUE_END
+};
+
static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
diff --git a/mali_kbase/mali_kbase.h b/mali_kbase/mali_kbase.h
index 66e4349..0445e0c 100644
--- a/mali_kbase/mali_kbase.h
+++ b/mali_kbase/mali_kbase.h
@@ -56,10 +56,10 @@
*/
#include "mali_kbase_defs.h"
+#include "debug/mali_kbase_debug_ktrace.h"
#include "context/mali_kbase_context.h"
#include "mali_kbase_strings.h"
#include "mali_kbase_mem_lowlevel.h"
-#include "mali_kbase_js.h"
#include "mali_kbase_utility.h"
#include "mali_kbase_mem.h"
#include "mmu/mali_kbase_mmu.h"
@@ -70,6 +70,7 @@
#include "mali_kbase_debug_job_fault.h"
#include "mali_kbase_jd_debugfs.h"
#include "mali_kbase_jm.h"
+#include "mali_kbase_js.h"
#include "ipa/mali_kbase_ipa.h"
@@ -161,6 +162,25 @@ void kbase_sysfs_term(struct kbase_device *kbdev);
int kbase_protected_mode_init(struct kbase_device *kbdev);
void kbase_protected_mode_term(struct kbase_device *kbdev);
+/**
+ * kbase_device_pm_init() - Performs power management initialization and
+ * Verifies device tree configurations.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Return: 0 if successful, otherwise a standard Linux error code
+ */
+int kbase_device_pm_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_device_pm_term() - Performs power management deinitialization and
+ * Free resources.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Clean up all the resources
+ */
+void kbase_device_pm_term(struct kbase_device *kbdev);
+
+
int power_control_init(struct kbase_device *kbdev);
void power_control_term(struct kbase_device *kbdev);
@@ -385,6 +405,24 @@ static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev)
return kbdev->pm.suspending;
}
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+/*
+ * Check whether a gpu lost is in progress
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Indicates whether a gpu lost has been received and jobs are no longer
+ * being scheduled
+ *
+ * Return: false if gpu is lost
+ * Return: != false otherwise
+ */
+static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev)
+{
+ return kbdev->pm.gpu_lost;
+}
+#endif
+
/**
* kbase_pm_is_active - Determine whether the GPU is active
*
@@ -534,181 +572,6 @@ void kbase_disjoint_state_down(struct kbase_device *kbdev);
#define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
#endif
-#if KBASE_TRACE_ENABLE
-void kbasep_trace_debugfs_init(struct kbase_device *kbdev);
-
-#ifndef CONFIG_MALI_SYSTEM_TRACE
-/** Add trace values about a job-slot
- *
- * @note Any functions called through this macro will still be evaluated in
- * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
- * functions called to get the parameters supplied to this macro must:
- * - be static or static inline
- * - must just return 0 and have no other statements present in the body.
- */
-#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \
- kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
- KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0)
-
-/** Add trace values about a job-slot, with info
- *
- * @note Any functions called through this macro will still be evaluated in
- * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
- * functions called to get the parameters supplied to this macro must:
- * - be static or static inline
- * - must just return 0 and have no other statements present in the body.
- */
-#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \
- kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
- KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val)
-
-/** Add trace values about a ctx refcount
- *
- * @note Any functions called through this macro will still be evaluated in
- * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
- * functions called to get the parameters supplied to this macro must:
- * - be static or static inline
- * - must just return 0 and have no other statements present in the body.
- */
-#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \
- kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
- KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0)
-/** Add trace values about a ctx refcount, and info
- *
- * @note Any functions called through this macro will still be evaluated in
- * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
- * functions called to get the parameters supplied to this macro must:
- * - be static or static inline
- * - must just return 0 and have no other statements present in the body.
- */
-#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \
- kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
- KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val)
-
-/** Add trace values (no slot or refcount)
- *
- * @note Any functions called through this macro will still be evaluated in
- * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
- * functions called to get the parameters supplied to this macro must:
- * - be static or static inline
- * - must just return 0 and have no other statements present in the body.
- */
-#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \
- kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
- 0, 0, 0, info_val)
-
-/** Clear the trace */
-#define KBASE_TRACE_CLEAR(kbdev) \
- kbasep_trace_clear(kbdev)
-
-/** Dump the slot trace */
-#define KBASE_TRACE_DUMP(kbdev) \
- kbasep_trace_dump(kbdev)
-
-/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */
-void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val);
-/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */
-void kbasep_trace_clear(struct kbase_device *kbdev);
-#else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
-/* Dispatch kbase trace events as system trace events */
-#include <mali_linux_kbase_trace.h>
-#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
- trace_mali_##code(jobslot, 0)
-
-#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
- trace_mali_##code(jobslot, info_val)
-
-#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
- trace_mali_##code(refcount, 0)
-
-#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
- trace_mali_##code(refcount, info_val)
-
-#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\
- trace_mali_##code(gpu_addr, info_val)
-
-#define KBASE_TRACE_CLEAR(kbdev)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(0);\
- } while (0)
-#define KBASE_TRACE_DUMP(kbdev)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(0);\
- } while (0)
-
-#endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */
-#else
-#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(code);\
- CSTD_UNUSED(ctx);\
- CSTD_UNUSED(katom);\
- CSTD_UNUSED(gpu_addr);\
- CSTD_UNUSED(jobslot);\
- } while (0)
-
-#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(code);\
- CSTD_UNUSED(ctx);\
- CSTD_UNUSED(katom);\
- CSTD_UNUSED(gpu_addr);\
- CSTD_UNUSED(jobslot);\
- CSTD_UNUSED(info_val);\
- CSTD_NOP(0);\
- } while (0)
-
-#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(code);\
- CSTD_UNUSED(ctx);\
- CSTD_UNUSED(katom);\
- CSTD_UNUSED(gpu_addr);\
- CSTD_UNUSED(refcount);\
- CSTD_NOP(0);\
- } while (0)
-
-#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(code);\
- CSTD_UNUSED(ctx);\
- CSTD_UNUSED(katom);\
- CSTD_UNUSED(gpu_addr);\
- CSTD_UNUSED(info_val);\
- CSTD_NOP(0);\
- } while (0)
-
-#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(code);\
- CSTD_UNUSED(subcode);\
- CSTD_UNUSED(ctx);\
- CSTD_UNUSED(katom);\
- CSTD_UNUSED(val);\
- CSTD_NOP(0);\
- } while (0)
-
-#define KBASE_TRACE_CLEAR(kbdev)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(0);\
- } while (0)
-#define KBASE_TRACE_DUMP(kbdev)\
- do {\
- CSTD_UNUSED(kbdev);\
- CSTD_NOP(0);\
- } while (0)
-#endif /* KBASE_TRACE_ENABLE */
-/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */
-void kbasep_trace_dump(struct kbase_device *kbdev);
-
#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI)
/* kbase_io_history_init - initialize data struct for register access history
@@ -758,5 +621,4 @@ int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size);
#endif /* CONFIG_DEBUG_FS */
-
#endif
diff --git a/mali_kbase/mali_kbase_core_linux.c b/mali_kbase/mali_kbase_core_linux.c
index 3f3d5cc..fb2353e 100644
--- a/mali_kbase/mali_kbase_core_linux.c
+++ b/mali_kbase/mali_kbase_core_linux.c
@@ -59,12 +59,16 @@
#include "mali_kbase_hwcnt_virtualizer.h"
#include "mali_kbase_hwcnt_legacy.h"
#include "mali_kbase_vinstr.h"
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include "arbiter/mali_kbase_arbiter_pm.h"
+#endif
#include "mali_kbase_cs_experimental.h"
#ifdef CONFIG_MALI_CINSTR_GWT
#include "mali_kbase_gwt.h"
#endif
+#include "mali_kbase_pm_internal.h"
#include <linux/module.h>
#include <linux/init.h>
@@ -852,7 +856,7 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx,
union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo)
{
u32 flags = timeinfo->in.request_flags;
- struct timespec ts;
+ struct timespec64 ts;
u64 timestamp;
u64 cycle_cnt;
@@ -2432,11 +2436,16 @@ struct kbasep_debug_command {
kbasep_debug_command_func *func;
};
+void kbasep_ktrace_dump_wrapper(struct kbase_device *kbdev)
+{
+ KBASE_KTRACE_DUMP(kbdev);
+}
+
/* Debug commands supported by the driver */
static const struct kbasep_debug_command debug_commands[] = {
{
.str = "dumptrace",
- .func = &kbasep_trace_dump,
+ .func = &kbasep_ktrace_dump_wrapper,
}
};
@@ -2558,9 +2567,9 @@ static ssize_t kbase_show_gpuinfo(struct device *dev,
{ .id = GPU_ID2_PRODUCT_TTRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
.name = "Mali-G77" },
{ .id = GPU_ID2_PRODUCT_TBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
- .name = "Mali-TBEX" },
+ .name = "Mali-G78" },
{ .id = GPU_ID2_PRODUCT_LBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
- .name = "Mali-LBEX" },
+ .name = "Mali-G68" },
{ .id = GPU_ID2_PRODUCT_TNAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
.name = "Mali-G57" },
{ .id = GPU_ID2_PRODUCT_TODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT,
@@ -3373,6 +3382,124 @@ void registers_unmap(struct kbase_device *kbdev)
kbase_common_reg_unmap(kbdev);
}
+#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
+
+static bool kbase_is_pm_enabled(const struct device_node *gpu_node)
+{
+ const struct device_node *power_model_node;
+ const void *cooling_cells_node;
+ const void *operating_point_node;
+ bool is_pm_enable = false;
+
+ power_model_node = of_get_child_by_name(gpu_node,
+ "power_model");
+ if (power_model_node)
+ is_pm_enable = true;
+
+ cooling_cells_node = of_get_property(gpu_node,
+ "#cooling-cells", NULL);
+ if (cooling_cells_node)
+ is_pm_enable = true;
+
+ operating_point_node = of_get_property(gpu_node,
+ "operating-points", NULL);
+ if (operating_point_node)
+ is_pm_enable = true;
+
+ return is_pm_enable;
+}
+
+static bool kbase_is_pv_enabled(const struct device_node *gpu_node)
+{
+ const void *arbiter_if_node;
+
+ arbiter_if_node = of_get_property(gpu_node,
+ "arbiter_if", NULL);
+
+ return arbiter_if_node ? true : false;
+}
+
+static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node)
+{
+ const void *coherency_dts;
+ u32 coherency;
+
+ coherency_dts = of_get_property(gpu_node,
+ "system-coherency",
+ NULL);
+ if (coherency_dts) {
+ coherency = be32_to_cpup(coherency_dts);
+ if (coherency == COHERENCY_ACE)
+ return true;
+ }
+ return false;
+}
+
+#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */
+
+int kbase_device_pm_init(struct kbase_device *kbdev)
+{
+ int err = 0;
+
+#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
+
+ u32 gpu_id;
+ u32 product_id;
+ u32 gpu_model_id;
+
+ if (kbase_is_pv_enabled(kbdev->dev->of_node)) {
+ if (kbase_is_pm_enabled(kbdev->dev->of_node)) {
+ /* Arbitration AND power management invalid */
+ dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n");
+ return -EPERM;
+ }
+ if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) {
+ /* Arbitration AND full coherency invalid */
+ dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n");
+ return -EPERM;
+ }
+ err = kbase_arbiter_pm_early_init(kbdev);
+ if (err == 0) {
+ /* Check if Arbitration is running on
+ * supported GPU platform
+ */
+ kbase_pm_register_access_enable(kbdev);
+ gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
+ kbase_pm_register_access_disable(kbdev);
+ product_id = KBASE_UBFX32(gpu_id,
+ GPU_ID_VERSION_PRODUCT_ID_SHIFT, 16);
+ gpu_model_id = GPU_ID2_MODEL_MATCH_VALUE(product_id);
+
+ if (gpu_model_id != GPU_ID2_PRODUCT_TGOX
+ && gpu_model_id != GPU_ID2_PRODUCT_TNOX) {
+ kbase_arbiter_pm_early_term(kbdev);
+ dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n");
+ return -EPERM;
+ }
+ }
+ } else {
+ err = power_control_init(kbdev);
+ }
+#else
+ err = power_control_init(kbdev);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */
+ return err;
+}
+
+void kbase_device_pm_term(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#ifdef CONFIG_OF
+ if (kbase_is_pv_enabled(kbdev->dev->of_node))
+ kbase_arbiter_pm_early_term(kbdev);
+ else
+ power_control_term(kbdev);
+#endif /* CONFIG_OF */
+#else
+ power_control_term(kbdev);
+#endif
+}
+
int power_control_init(struct kbase_device *kbdev)
{
#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF)
@@ -3770,9 +3897,7 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev)
kbdev->mali_debugfs_directory, kbdev,
&fops_trigger_reset);
-#if KBASE_TRACE_ENABLE
- kbasep_trace_debugfs_init(kbdev);
-#endif /* KBASE_TRACE_ENABLE */
+ kbase_ktrace_debugfs_init(kbdev);
#ifdef CONFIG_MALI_DEVFREQ
#ifdef CONFIG_DEVFREQ_THERMAL
@@ -4006,6 +4131,11 @@ static int kbase_platform_device_probe(struct platform_device *pdev)
"Probed as %s\n", dev_name(kbdev->mdev.this_device));
#endif /* MALI_KBASE_BUILD */
kbase_increment_device_id();
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ mutex_lock(&kbdev->pm.lock);
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT);
+ mutex_unlock(&kbdev->pm.lock);
+#endif
}
return err;
@@ -4248,7 +4378,6 @@ MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \
__stringify(BASE_UK_VERSION_MAJOR) "." \
__stringify(BASE_UK_VERSION_MINOR) ")");
-
#define CREATE_TRACE_POINTS
/* Create the trace points (otherwise we just get code to call a tracepoint) */
#include "mali_linux_trace.h"
@@ -4282,6 +4411,3 @@ void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event)
trace_mali_total_alloc_pages_change(dev_id, event);
}
#endif /* CONFIG_MALI_GATOR_SUPPORT */
-#ifdef CONFIG_MALI_SYSTEM_TRACE
-#include "mali_linux_kbase_trace.h"
-#endif
diff --git a/mali_kbase/mali_kbase_ctx_sched.c b/mali_kbase/mali_kbase_ctx_sched.c
index 3922260..cea91bc 100644
--- a/mali_kbase/mali_kbase_ctx_sched.c
+++ b/mali_kbase/mali_kbase_ctx_sched.c
@@ -23,7 +23,23 @@
#include <mali_kbase.h>
#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_defs.h>
#include "mali_kbase_ctx_sched.h"
+#include "tl/mali_kbase_tracepoints.h"
+
+/* Helper for ktrace */
+#if KBASE_KTRACE_ENABLE
+static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
+{
+ return atomic_read(&kctx->refcount);
+}
+#else /* KBASE_KTRACE_ENABLE */
+static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
+{
+ CSTD_UNUSED(kctx);
+ return 0;
+}
+#endif /* KBASE_KTRACE_ENABLE */
int kbase_ctx_sched_init(struct kbase_device *kbdev)
{
@@ -106,11 +122,15 @@ int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx)
if (prev_kctx) {
WARN_ON(atomic_read(&prev_kctx->refcount) != 0);
kbase_mmu_disable(prev_kctx);
+ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
+ kbdev, prev_kctx->id);
prev_kctx->as_nr = KBASEP_AS_NR_INVALID;
}
kctx->as_nr = free_as;
kbdev->as_to_kctx[free_as] = kctx;
+ KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS(
+ kbdev, kctx->id, free_as);
kbase_mmu_update(kbdev, &kctx->mmu,
kctx->as_nr);
}
@@ -142,17 +162,23 @@ void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx)
void kbase_ctx_sched_release_ctx(struct kbase_context *kctx)
{
struct kbase_device *const kbdev = kctx->kbdev;
+ int new_ref_count;
lockdep_assert_held(&kbdev->hwaccess_lock);
- if (atomic_dec_return(&kctx->refcount) == 0) {
+ new_ref_count = atomic_dec_return(&kctx->refcount);
+ if (new_ref_count == 0) {
kbdev->as_free |= (1u << kctx->as_nr);
if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) {
+ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
+ kbdev, kctx->id);
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT);
}
}
+
+ KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count);
}
void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx)
@@ -168,6 +194,7 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx)
if (kbdev->pm.backend.gpu_powered)
kbase_mmu_disable(kctx);
+ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(kbdev, kctx->id);
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
}
@@ -198,11 +225,120 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev)
/* This context might have been assigned an
* AS before, clear it.
*/
- kbdev->as_to_kctx[kctx->as_nr] = NULL;
- kctx->as_nr = KBASEP_AS_NR_INVALID;
+ if (kctx->as_nr != KBASEP_AS_NR_INVALID) {
+ KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
+ kbdev, kctx->id);
+ kbdev->as_to_kctx[kctx->as_nr] = NULL;
+ kctx->as_nr = KBASEP_AS_NR_INVALID;
+ }
}
} else {
kbase_mmu_disable_as(kbdev, i);
}
}
}
+
+struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount(
+ struct kbase_device *kbdev, size_t as_nr)
+{
+ unsigned long flags;
+ struct kbase_context *found_kctx = NULL;
+
+ if (WARN_ON(kbdev == NULL))
+ return NULL;
+
+ if (WARN_ON(as_nr >= BASE_MAX_NR_AS))
+ return NULL;
+
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+
+ found_kctx = kbdev->as_to_kctx[as_nr];
+
+ if (found_kctx != NULL)
+ kbase_ctx_sched_retain_ctx_refcount(found_kctx);
+
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+ return found_kctx;
+}
+
+struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev,
+ size_t as_nr)
+{
+ struct kbase_context *found_kctx;
+
+ if (WARN_ON(kbdev == NULL))
+ return NULL;
+
+ if (WARN_ON(as_nr >= BASE_MAX_NR_AS))
+ return NULL;
+
+ found_kctx = kbdev->as_to_kctx[as_nr];
+
+ if (WARN_ON(!found_kctx))
+ return NULL;
+
+ if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0))
+ return NULL;
+
+ return found_kctx;
+}
+
+bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx)
+{
+ bool result = false;
+ int as_nr;
+
+ if (WARN_ON(kctx == NULL))
+ return result;
+
+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
+
+ as_nr = kctx->as_nr;
+ if (atomic_read(&kctx->refcount) > 0) {
+ KBASE_DEBUG_ASSERT(as_nr >= 0);
+
+ kbase_ctx_sched_retain_ctx_refcount(kctx);
+ KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx,
+ kbase_ktrace_get_ctx_refcnt(kctx));
+ result = true;
+ }
+
+ return result;
+}
+
+bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx)
+{
+ unsigned long flags;
+ bool result = false;
+
+ if (WARN_ON(kctx == NULL))
+ return result;
+
+ if (WARN_ON(kctx->kbdev == NULL))
+ return result;
+
+ mutex_lock(&kctx->kbdev->mmu_hw_mutex);
+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
+ result = kbase_ctx_sched_inc_refcount_nolock(kctx);
+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
+ mutex_unlock(&kctx->kbdev->mmu_hw_mutex);
+
+ return result;
+}
+
+void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx)
+{
+ unsigned long flags;
+
+ if (WARN_ON(!kctx))
+ return;
+
+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
+
+ if (!WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID) &&
+ !WARN_ON(atomic_read(&kctx->refcount) <= 0))
+ kbase_ctx_sched_release_ctx(kctx);
+
+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
+}
diff --git a/mali_kbase/mali_kbase_ctx_sched.h b/mali_kbase/mali_kbase_ctx_sched.h
index ab57a0d..1affa71 100644
--- a/mali_kbase/mali_kbase_ctx_sched.h
+++ b/mali_kbase/mali_kbase_ctx_sched.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -132,4 +132,78 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx);
*/
void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev);
+/**
+ * kbase_ctx_sched_as_to_ctx_refcount - Lookup a context based on its current
+ * address space and ensure that is stays scheduled in
+ * @kbdev: The device for which the returned context must belong
+ * @as_nr: address space assigned to the context of interest
+ *
+ * The context is refcounted as being busy to prevent it from scheduling
+ * out. It must be released with kbase_ctx_sched_release_ctx() when it is no
+ * longer required to stay scheduled in.
+ *
+ * This function can safely be called from IRQ context.
+ *
+ * The following locking conditions are made on the caller:
+ * * it must not hold the kbase_device::hwaccess_lock, because it will be used
+ * internally.
+ *
+ * Return: a valid struct kbase_context on success, which has been refcounted
+ * as being busy or return NULL on failure, indicating that no context was found
+ * in as_nr.
+ */
+struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount(
+ struct kbase_device *kbdev, size_t as_nr);
+
+/**
+ * kbase_ctx_sched_as_to_ctx - Lookup a context based on its current address
+ * space
+ * @kbdev: The device for which the returned context must belong
+ * @as_nr: address space assigned to the context of interest
+ *
+ * Return: a valid struct kbase_context on success or NULL on failure,
+ * indicating that no context was found in as_nr.
+ */
+struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev,
+ size_t as_nr);
+
+/**
+ * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy,
+ * preventing it from being scheduled out.
+ * @kctx: Context to be refcounted
+ *
+ * The following locks must be held by the caller:
+ * * kbase_device::mmu_hw_mutex
+ * * kbase_device::hwaccess_lock
+ *
+ * Return: true if refcount succeeded, and the context will not be scheduled
+ * out, false if the refcount failed (because the context is being/has been
+ * scheduled out).
+ */
+bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx);
+
+/**
+ * kbase_ctx_sched_inc_refcount - Refcount a context as being busy, preventing
+ * it from being scheduled out.
+ * @kctx: Context to be refcounted
+ *
+ * The following locking conditions are made on the caller:
+ * * it must not hold kbase_device::mmu_hw_mutex and
+ * kbase_device::hwaccess_lock, because they will be used internally.
+ *
+ * Return: true if refcount succeeded, and the context will not be scheduled
+ * out, false if the refcount failed (because the context is being/has been
+ * scheduled out).
+ */
+bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx);
+
+/**
+ * kbase_ctx_sched_release_ctx_lock - Release a reference count of a context
+ * @kctx: Context for which refcount should be decreased
+ *
+ * Effectivelly, this is a wrapper for kbase_ctx_sched_release_ctx, but
+ * kbase_device::hwaccess_lock is required NOT to be locked.
+ */
+void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx);
+
#endif /* _KBASE_CTX_SCHED_H_ */
diff --git a/mali_kbase/mali_kbase_defs.h b/mali_kbase/mali_kbase_defs.h
index ce32b53..7056d80 100644
--- a/mali_kbase/mali_kbase_defs.h
+++ b/mali_kbase/mali_kbase_defs.h
@@ -61,12 +61,16 @@
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
-#endif /* CONFIG_DEBUG_FS */
+#endif /* CONFIG_DEBUG_FS */
#ifdef CONFIG_MALI_DEVFREQ
#include <linux/devfreq.h>
#endif /* CONFIG_MALI_DEVFREQ */
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include <arbiter/mali_kbase_arbiter_defs.h>
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+
#include <linux/clk.h>
#include <linux/regulator/consumer.h>
#include <linux/memory_group_manager.h>
@@ -76,18 +80,7 @@
#define KBASE_PM_RUNTIME 1
#endif
-/** Enable SW tracing when set */
-#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE
-#define KBASE_TRACE_ENABLE 1
-#endif
-
-#ifndef KBASE_TRACE_ENABLE
-#ifdef CONFIG_MALI_DEBUG
-#define KBASE_TRACE_ENABLE 1
-#else
-#define KBASE_TRACE_ENABLE 0
-#endif /* CONFIG_MALI_DEBUG */
-#endif /* KBASE_TRACE_ENABLE */
+#include "debug/mali_kbase_debug_ktrace_defs.h"
/** Number of milliseconds before we time out on a GPU soft/hard reset */
#define RESET_TIMEOUT 500
@@ -130,11 +123,6 @@
*/
#define KBASE_LOCK_REGION_MIN_SIZE_LOG2 (15)
-#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */
-#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2)
-#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1)
-
-#include "mali_kbase_js_defs.h"
#include "mali_kbase_hwaccess_defs.h"
/* Maximum number of pages of memory that require a permanent mapping, per
@@ -332,70 +320,6 @@ struct kbasep_mem_device {
atomic_t ir_threshold;
};
-#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X
-
-enum kbase_trace_code {
- /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
- * THIS MUST BE USED AT THE START OF THE ENUM */
-#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X)
-#include <tl/mali_kbase_trace_defs.h>
-#undef KBASE_TRACE_CODE_MAKE_CODE
- /* Comma on its own, to extend the list */
- ,
- /* Must be the last in the enum */
- KBASE_TRACE_CODE_COUNT
-};
-
-#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0)
-#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1)
-
-/**
- * struct kbase_trace - object representing a trace message added to trace buffer
- * kbase_device::trace_rbuf
- * @timestamp: CPU timestamp at which the trace message was added.
- * @thread_id: id of the thread in the context of which trace message
- * was added.
- * @cpu: indicates which CPU the @thread_id was scheduled on when
- * the trace message was added.
- * @ctx: Pointer to the kbase context for which the trace message
- * was added. Will be NULL for certain trace messages like
- * for traces added corresponding to power management events.
- * Will point to the appropriate context corresponding to
- * job-slot & context's reference count related events.
- * @katom: indicates if the trace message has atom related info.
- * @atom_number: id of the atom for which trace message was added.
- * Only valid if @katom is true.
- * @atom_udata: Copy of the user data sent for the atom in base_jd_submit.
- * Only valid if @katom is true.
- * @gpu_addr: GPU address of the job-chain represented by atom. Could
- * be valid even if @katom is false.
- * @info_val: value specific to the type of event being traced. For the
- * case where @katom is true, will be set to atom's affinity,
- * i.e. bitmask of shader cores chosen for atom's execution.
- * @code: Identifies the event, refer enum kbase_trace_code.
- * @jobslot: job-slot for which trace message was added, valid only for
- * job-slot management events.
- * @refcount: reference count for the context, valid for certain events
- * related to scheduler core and policy.
- * @flags: indicates if info related to @jobslot & @refcount is present
- * in the trace message, used during dumping of the message.
- */
-struct kbase_trace {
- struct timespec timestamp;
- u32 thread_id;
- u32 cpu;
- void *ctx;
- bool katom;
- int atom_number;
- u64 atom_udata[2];
- u64 gpu_addr;
- unsigned long info_val;
- u8 code;
- u8 jobslot;
- u8 refcount;
- u8 flags;
-};
-
/**
* Data stored per device for power management.
*
@@ -420,6 +344,10 @@ struct kbase_pm_device_data {
int active_count;
/** Flag indicating suspending/suspended */
bool suspending;
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ /* Flag indicating gpu lost */
+ bool gpu_lost;
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
/* Wait queue set when active_count == 0 */
wait_queue_head_t zero_active_count_wait;
@@ -450,6 +378,13 @@ struct kbase_pm_device_data {
u32 dvfs_period;
struct kbase_pm_backend_data backend;
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ /**
+ * The state of the arbiter VM machine
+ */
+ struct kbase_arbiter_vm_state *arb_vm_state;
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
};
/**
@@ -722,8 +657,9 @@ struct kbase_devfreq_queue_info {
* kbase_hwcnt_context_enable() with @hwcnt_gpu_ctx.
* @hwcnt_gpu_virt: Virtualizer for GPU hardware counters.
* @vinstr_ctx: vinstr context created per device.
- * @timeline_is_enabled: Non zero, if there is at least one timeline client,
- * zero otherwise.
+ * @timeline_flags: Bitmask defining which sets of timeline tracepoints
+ * are enabled. If zero, there is no timeline client and
+ * therefore timeline is disabled.
* @timeline: Timeline context created per device.
* @trace_lock: Lock to serialize the access to trace buffer.
* @trace_first_out: Index/offset in the trace buffer at which the first
@@ -913,7 +849,6 @@ struct kbase_device {
struct kbase_pm_device_data pm;
- struct kbasep_js_device_data js_data;
struct kbase_mem_pool_group mem_pools;
struct kbasep_mem_device memdev;
struct kbase_mmu_mode const *mmu_mode;
@@ -955,14 +890,11 @@ struct kbase_device {
struct kbase_hwcnt_virtualizer *hwcnt_gpu_virt;
struct kbase_vinstr_context *vinstr_ctx;
- atomic_t timeline_is_enabled;
+ atomic_t timeline_flags;
struct kbase_timeline *timeline;
-#if KBASE_TRACE_ENABLE
- spinlock_t trace_lock;
- u16 trace_first_out;
- u16 trace_next_in;
- struct kbase_trace *trace_rbuf;
+#if KBASE_KTRACE_TARGET_RBUF
+ struct kbase_ktrace ktrace;
#endif
u32 reset_timeout_ms;
@@ -1098,6 +1030,8 @@ struct kbase_device {
u8 l2_size_override;
u8 l2_hash_override;
+ struct kbasep_js_device_data js_data;
+
/* See KBASE_JS_*_PRIORITY_MODE for details. */
u32 js_ctx_scheduling_mode;
@@ -1115,6 +1049,11 @@ struct kbase_device {
int slot;
u64 flags;
} dummy_job_wa;
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ /* Pointer to the arbiter device */
+ struct kbase_arbiter_device arb;
+#endif
};
#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \
@@ -1478,12 +1417,6 @@ struct kbase_sub_alloc {
* @jit_work: Work item queued to defer the freeing of a memory
* region when a just-in-time memory allocation is moved
* to @jit_destroy_head.
- * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
- * allocate & free, in submission order, protected by
- * &struct_kbase_jd_context.lock.
- * @jit_pending_alloc: A list of just-in-time memory allocation soft-jobs
- * which will be reattempted after the impending free of
- * other active allocations.
* @ext_res_meta_head: A list of sticky external resources which were requested to
* be mapped on GPU side, through a softjob atom of type
* EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl.
@@ -1500,7 +1433,7 @@ struct kbase_sub_alloc {
* @gwt_snapshot_list: Snapshot of the @gwt_current_list for sending to user space.
* @priority: Indicates the context priority. Used along with @atoms_count
* for context scheduling, protected by hwaccess_lock.
- * @atoms_count: Number of gpu atoms currently in use, per priority
+ * @atoms_count: Number of GPU atoms currently in use, per priority
* @create_flags: Flags used in context creation.
*
* A kernel base context is an entity among which the GPU is scheduled.
@@ -1563,7 +1496,7 @@ struct kbase_context {
pid_t tgid;
pid_t pid;
atomic_t used_pages;
- atomic_t nonmapped_pages;
+ atomic_t nonmapped_pages;
atomic_t permanent_mapped_pages;
struct kbase_mem_pool_group mem_pools;
@@ -1622,9 +1555,6 @@ struct kbase_context {
struct mutex jit_evict_lock;
struct work_struct jit_work;
- struct list_head jit_atoms_head;
- struct list_head jit_pending_alloc;
-
struct list_head ext_res_meta_head;
u8 trim_level;
diff --git a/mali_kbase/mali_kbase_dummy_job_wa.c b/mali_kbase/mali_kbase_dummy_job_wa.c
index 5830e8e..188e53b 100644
--- a/mali_kbase/mali_kbase_dummy_job_wa.c
+++ b/mali_kbase/mali_kbase_dummy_job_wa.c
@@ -257,74 +257,15 @@ static ssize_t show_dummy_job_wa_info(struct device * const dev,
static DEVICE_ATTR(dummy_job_wa_info, 0444, show_dummy_job_wa_info, NULL);
-#define FAIL_PROBE 1
-#define SKIP_WA 2
-#define LOAD_WA 3
-
-static int check_wa_validity(struct kbase_device *kbdev,
- bool wa_blob_present)
+static bool wa_blob_load_needed(struct kbase_device *kbdev)
{
- struct base_gpu_props *gpu_props = &kbdev->gpu_props.props;
- const u32 major_revision = gpu_props->core_props.major_revision;
- const u32 minor_revision = gpu_props->core_props.minor_revision;
- const u32 gpu_id = gpu_props->raw_props.gpu_id;
- const u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
- GPU_ID_VERSION_PRODUCT_ID_SHIFT;
- int ret = FAIL_PROBE;
-
- if (IS_ENABLED(CONFIG_ARCH_VEXPRESS))
- return SKIP_WA;
-
- switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) {
- case GPU_ID2_PRODUCT_TTRX:
- /* WA needed for r0p0, r0p1 only */
- if (major_revision == 0) {
- if ((minor_revision <= 1) && wa_blob_present)
- ret = LOAD_WA;
- else if ((minor_revision > 1) && !wa_blob_present)
- ret = SKIP_WA;
- } else if ((major_revision > 0) && !wa_blob_present)
- ret = SKIP_WA;
- break;
- case GPU_ID2_PRODUCT_TNAX:
- /* WA needed for r0p0 only */
- if (major_revision == 0) {
- if ((minor_revision == 0) && wa_blob_present)
- ret = LOAD_WA;
- else if ((minor_revision > 0) && !wa_blob_present)
- ret = SKIP_WA;
- } else if ((major_revision > 0) && !wa_blob_present)
- ret = SKIP_WA;
- break;
- case GPU_ID2_PRODUCT_TBEX:
- /* WA needed for r0p0 only */
- if ((major_revision == 0) && (minor_revision == 0)) {
- if (!wa_blob_present) {
- dev_warn(kbdev->dev, "Dummy job WA not applied, susceptible to GPU hang. Contact support-mali@arm.com");
- ret = SKIP_WA;
- } else
- ret = LOAD_WA;
- } else if (!wa_blob_present)
- ret = SKIP_WA;
- break;
- case GPU_ID2_PRODUCT_LBEX:
- /* WA needed for r1p0 only */
- if ((major_revision == 1) && (minor_revision == 0)) {
- if (!wa_blob_present) {
- dev_warn(kbdev->dev, "Dummy job WA not applied, susceptible to GPU hang. Contact support-mali@arm.com");
- ret = SKIP_WA;
- } else
- ret = LOAD_WA;
- } else if (!wa_blob_present)
- ret = SKIP_WA;
- break;
- default:
- if (!wa_blob_present)
- ret = SKIP_WA;
- break;
- }
+ if (of_machine_is_compatible("arm,juno"))
+ return false;
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485))
+ return true;
- return ret;
+ return false;
}
int kbase_dummy_job_wa_load(struct kbase_device *kbdev)
@@ -339,31 +280,17 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev)
const struct wa_v2_info *v2_info;
u32 blob_offset;
int err;
- int ret;
struct kbase_context *kctx;
+ if (!wa_blob_load_needed(kbdev))
+ return 0;
+
/* load the wa */
-#if KERNEL_VERSION(4, 18, 0) <= LINUX_VERSION_CODE
- err = firmware_request_nowarn(&firmware, wa_name, kbdev->dev);
-#else
err = request_firmware(&firmware, wa_name, kbdev->dev);
-#endif
- ret = check_wa_validity(kbdev, err == 0);
-
- if (ret == SKIP_WA) {
- if (err == 0)
- release_firmware(firmware);
- return 0;
- } else if (ret == FAIL_PROBE) {
- if (err == 0) {
- dev_err(kbdev->dev, "WA blob unexpectedly present. Please refer to the Arm Mali DDK Bifrost/Valhall Release Notes, "
- "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed");
- release_firmware(firmware);
- } else {
- dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, "
- "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed");
- }
+ if (err) {
+ dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, "
+ "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed");
return -ENODEV;
}
@@ -504,8 +431,9 @@ void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev)
sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr);
wa_ctx = READ_ONCE(kbdev->dummy_job_wa.ctx);
+ WRITE_ONCE(kbdev->dummy_job_wa.ctx, NULL);
/* make this write visible before we tear down the ctx */
- smp_store_mb(kbdev->dummy_job_wa.ctx, NULL);
+ smp_mb();
if (wa_ctx) {
kbasep_js_release_privileged_ctx(kbdev, wa_ctx);
diff --git a/mali_kbase/mali_kbase_event.c b/mali_kbase/mali_kbase_event.c
index 2bbc313..c8b8f22 100644
--- a/mali_kbase/mali_kbase_event.c
+++ b/mali_kbase/mali_kbase_event.c
@@ -25,6 +25,7 @@
#include <mali_kbase.h>
#include <mali_kbase_debug.h>
#include <tl/mali_kbase_tracepoints.h>
+#include <mali_linux_trace.h>
static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
@@ -200,6 +201,10 @@ void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom)
dev_dbg(kbdev->dev, "Reporting %d events\n", event_count);
kbase_event_wakeup(ctx);
+
+ /* Post-completion latency */
+ trace_sysgraph(SGR_POST, ctx->id,
+ kbase_jd_atom_id(ctx, atom));
}
}
KBASE_EXPORT_TEST_API(kbase_event_post);
diff --git a/mali_kbase/mali_kbase_gpuprops.c b/mali_kbase/mali_kbase_gpuprops.c
index f1f188f..ae2458f 100644
--- a/mali_kbase/mali_kbase_gpuprops.c
+++ b/mali_kbase/mali_kbase_gpuprops.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved.
@@ -37,19 +38,6 @@
#include <linux/of_platform.h>
#include <linux/moduleparam.h>
-/**
- * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield.
- * @value: The value from which to extract bits.
- * @offset: The first bit to extract (0 being the LSB).
- * @size: The number of bits to extract.
- *
- * Context: @offset + @size <= 32.
- *
- * Return: Bits [@offset, @offset + @size) from @value.
- */
-/* from mali_cdsb.h */
-#define KBASE_UBFX32(value, offset, size) \
- (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1))
static void kbase_gpuprops_construct_coherent_groups(
struct base_gpu_props * const props)
@@ -126,18 +114,23 @@ static void kbase_gpuprops_construct_coherent_groups(
*
* Fill the &struct base_gpu_props structure with values from the GPU
* configuration registers. Only the raw properties are filled in this function.
+ *
+ * Return: Zero on success, Linux error code on failure
*/
-static void kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props,
+static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props,
struct kbase_device *kbdev)
{
struct kbase_gpuprops_regdump regdump;
int i;
+ int err;
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != gpu_props);
/* Dump relevant registers */
- kbase_backend_gpuprops_get(kbdev, &regdump);
+ err = kbase_backend_gpuprops_get(kbdev, &regdump);
+ if (err)
+ return err;
gpu_props->raw_props.gpu_id = regdump.gpu_id;
gpu_props->raw_props.tiler_features = regdump.tiler_features;
@@ -172,6 +165,8 @@ static void kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props,
gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size;
gpu_props->raw_props.thread_features = regdump.thread_features;
gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc;
+
+ return 0;
}
void kbase_gpuprops_update_core_props_gpu_id(
@@ -325,15 +320,18 @@ void kbase_gpuprops_set(struct kbase_device *kbdev)
gpu_props->num_job_slots = hweight32(raw->js_present);
}
-void kbase_gpuprops_set_features(struct kbase_device *kbdev)
+int kbase_gpuprops_set_features(struct kbase_device *kbdev)
{
struct base_gpu_props *gpu_props;
struct kbase_gpuprops_regdump regdump;
+ int err;
gpu_props = &kbdev->gpu_props.props;
/* Dump relevant registers */
- kbase_backend_gpuprops_get_features(kbdev, &regdump);
+ err = kbase_backend_gpuprops_get_features(kbdev, &regdump);
+ if (err)
+ return err;
/*
* Copy the raw value from the register, later this will get turned
@@ -345,6 +343,8 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev)
if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT))
gpu_props->thread_props.max_thread_group_split = 0;
+
+ return err;
}
/*
@@ -396,15 +396,17 @@ static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev)
return false;
}
-void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
+int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
{
+ int err = 0;
+
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
struct kbase_gpuprops_regdump regdump;
struct base_gpu_props *gpu_props = &kbdev->gpu_props.props;
/* Check for L2 cache size & hash overrides */
if (!kbase_read_l2_config_from_dt(kbdev))
- return;
+ return 0;
/* Need L2 to get powered to reflect to L2_FEATURES */
kbase_pm_context_active(kbdev);
@@ -413,7 +415,9 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
kbase_pm_wait_for_l2_powered(kbdev);
/* Dump L2_FEATURES register */
- kbase_backend_gpuprops_get_l2_features(kbdev, &regdump);
+ err = kbase_backend_gpuprops_get_l2_features(kbdev, &regdump);
+ if (err)
+ goto idle_gpu;
dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n",
regdump.l2_features);
@@ -423,9 +427,12 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
gpu_props->l2_props.log2_cache_size =
KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8);
+idle_gpu:
/* Let GPU idle */
kbase_pm_context_idle(kbdev);
}
+
+ return err;
}
static struct {
diff --git a/mali_kbase/mali_kbase_gpuprops.h b/mali_kbase/mali_kbase_gpuprops.h
index eeba92f..5eee794 100644
--- a/mali_kbase/mali_kbase_gpuprops.h
+++ b/mali_kbase/mali_kbase_gpuprops.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -18,6 +18,20 @@
*
* SPDX-License-Identifier: GPL-2.0
*
+ *//* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *
+ * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
*/
@@ -36,6 +50,20 @@
struct kbase_device;
/**
+ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield.
+ * @value: The value from which to extract bits.
+ * @offset: The first bit to extract (0 being the LSB).
+ * @size: The number of bits to extract.
+ *
+ * Context: @offset + @size <= 32.
+ *
+ * Return: Bits [@offset, @offset + @size) from @value.
+ */
+/* from mali_cdsb.h */
+#define KBASE_UBFX32(value, offset, size) \
+ (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1))
+
+/**
* @brief Set up Kbase GPU properties.
*
* Set up Kbase GPU properties with information from the GPU registers
@@ -51,16 +79,20 @@ void kbase_gpuprops_set(struct kbase_device *kbdev);
* This function sets up GPU properties that are dependent on the hardware
* features bitmask. This function must be preceeded by a call to
* kbase_hw_set_features_mask().
+ *
+ * Return: Zero on success, Linux error code on failure
*/
-void kbase_gpuprops_set_features(struct kbase_device *kbdev);
+int kbase_gpuprops_set_features(struct kbase_device *kbdev);
/**
* kbase_gpuprops_update_l2_features - Update GPU property of L2_FEATURES
* @kbdev: Device pointer
*
* This function updates l2_features and the log2 cache size.
+ *
+ * Return: Zero on success, Linux error code for failure
*/
-void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev);
+int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev);
/**
* kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer
diff --git a/mali_kbase/mali_kbase_hw.c b/mali_kbase/mali_kbase_hw.c
index c3abad4..f8a9248 100644
--- a/mali_kbase/mali_kbase_hw.c
+++ b/mali_kbase/mali_kbase_hw.c
@@ -97,11 +97,11 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev)
for (; *features != BASE_HW_FEATURE_END; features++)
set_bit(*features, &kbdev->hw_features_mask[0]);
-#if defined(CONFIG_MALI_JOB_DUMP) || defined(CONFIG_MALI_VECTOR_DUMP)
+#if defined(CONFIG_MALI_VECTOR_DUMP)
/* When dumping is enabled, need to disable flush reduction optimization
* for GPUs on which it is safe to have only cache clean operation at
* the end of job chain.
- * This is required to make job dumping work. There is some discrepancy
+ * This is required to make vector dump work. There is some discrepancy
* in the implementation of flush reduction optimization due to
* unclear or ambiguous ARCH spec.
*/
@@ -195,14 +195,14 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(
{U32_MAX, NULL} } },
{GPU_ID2_PRODUCT_LBEX,
- {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0},
- {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tBEx_r1p1},
+ {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0},
+ {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1},
{U32_MAX, NULL} } },
{GPU_ID2_PRODUCT_TBEX,
{{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0},
{GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0},
- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p0},
+ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1},
{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0},
{U32_MAX, NULL} } },
diff --git a/mali_kbase/mali_kbase_hwaccess_gpuprops.h b/mali_kbase/mali_kbase_hwaccess_gpuprops.h
index 62628b6..3ae0dbe 100644
--- a/mali_kbase/mali_kbase_hwaccess_gpuprops.h
+++ b/mali_kbase/mali_kbase_hwaccess_gpuprops.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014-2015, 2018, 2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -18,6 +18,20 @@
*
* SPDX-License-Identifier: GPL-2.0
*
+ *//* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *
+ * (C) COPYRIGHT 2014-2015, 2018, 2019-2020 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
*/
@@ -35,8 +49,10 @@
* @regdump: Pointer to struct kbase_gpuprops_regdump structure
*
* The caller should ensure that GPU remains powered-on during this function.
+ *
+ * Return: Zero for succeess or a Linux error code
*/
-void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump);
/**
@@ -47,8 +63,10 @@ void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
*
* This function reads GPU properties that are dependent on the hardware
* features bitmask. It will power-on the GPU if required.
+ *
+ * Return: Zero for succeess or a Linux error code
*/
-void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump);
/**
@@ -59,8 +77,10 @@ void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
*
* This function reads L2_FEATURES register that is dependent on the hardware
* features bitmask. It will power-on the GPU if required.
+ *
+ * Return: Zero on success, Linux error code on failure
*/
-void kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
+int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump);
diff --git a/mali_kbase/mali_kbase_hwaccess_time.h b/mali_kbase/mali_kbase_hwaccess_time.h
index ca0cd79..a61e5b9 100644
--- a/mali_kbase/mali_kbase_hwaccess_time.h
+++ b/mali_kbase/mali_kbase_hwaccess_time.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2014,2018-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -37,6 +37,6 @@
* time in
*/
void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
- u64 *system_time, struct timespec *ts);
+ u64 *system_time, struct timespec64 *ts);
#endif /* _KBASE_BACKEND_TIME_H_ */
diff --git a/mali_kbase/mali_kbase_hwcnt_backend_gpu.c b/mali_kbase/mali_kbase_hwcnt_backend_gpu.c
index ae11630..407c768 100644
--- a/mali_kbase/mali_kbase_hwcnt_backend_gpu.c
+++ b/mali_kbase/mali_kbase_hwcnt_backend_gpu.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -73,11 +73,8 @@ struct kbase_hwcnt_backend_gpu {
static u64 kbasep_hwcnt_backend_gpu_timestamp_ns(
struct kbase_hwcnt_backend *backend)
{
- struct timespec ts;
-
(void)backend;
- getrawmonotonic(&ts);
- return (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
+ return ktime_get_raw_ns();
}
/* GPU backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */
diff --git a/mali_kbase/mali_kbase_jd.c b/mali_kbase/mali_kbase_jd.c
index b4ae3ba..1a830dd 100644
--- a/mali_kbase/mali_kbase_jd.c
+++ b/mali_kbase/mali_kbase_jd.c
@@ -34,6 +34,7 @@
#include <mali_kbase_jm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <tl/mali_kbase_tracepoints.h>
+#include <mali_linux_trace.h>
#include "mali_kbase_dma_fence.h"
#include <mali_kbase_cs_experimental.h>
@@ -94,6 +95,8 @@ static bool jd_run_atom(struct kbase_jd_atom *katom)
if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) {
/* Dependency only atom */
+ trace_sysgraph(SGR_SUBMIT, kctx->id,
+ kbase_jd_atom_id(katom->kctx, katom));
katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
dev_dbg(kctx->kbdev->dev, "Atom %p status to completed\n",
(void *)katom);
@@ -452,6 +455,9 @@ static inline void jd_resolve_dep(struct list_head *out_list,
#endif /* CONFIG_MALI_DMA_FENCE */
if (dep_satisfied) {
+ trace_sysgraph(SGR_DEP_RES,
+ dep_atom->kctx->id,
+ kbase_jd_atom_id(katom->kctx, dep_atom));
dep_atom->in_jd_list = true;
list_add_tail(&dep_atom->jd_item, out_list);
}
@@ -540,6 +546,10 @@ static void jd_try_submitting_deps(struct list_head *out_list,
#endif /* CONFIG_MALI_DMA_FENCE */
if (dep0_valid && dep1_valid && dep_satisfied) {
+ trace_sysgraph(SGR_DEP_RES,
+ dep_atom->kctx->id,
+ kbase_jd_atom_id(dep_atom->kctx,
+ dep_atom));
dep_atom->in_jd_list = true;
list_add(&dep_atom->jd_item, out_list);
}
@@ -576,6 +586,7 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom)
for (idx = 0;
idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx];
idx++) {
+ enum heap_pointer { LOW = 0, HIGH, COUNT };
size_t size_to_read;
u64 read_val;
@@ -601,6 +612,8 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom)
size_to_read = sizeof(*ptr);
if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)
size_to_read = sizeof(u32);
+ else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP)
+ size_to_read = sizeof(u64[COUNT]);
ptr = kbase_vmap(kctx, reg->heap_info_gpu_addr, size_to_read,
&mapping);
@@ -618,20 +631,41 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom)
read_val = READ_ONCE(*(u32 *)ptr);
used_pages = PFN_UP(read_val);
} else {
- u64 addr_end = read_val = READ_ONCE(*ptr);
+ u64 addr_end;
if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) {
- unsigned long extent_bytes = reg->extent <<
- PAGE_SHIFT;
+ const unsigned long extent_bytes = reg->extent
+ << PAGE_SHIFT;
+ const u64 low_ptr = ptr[LOW];
+ const u64 high_ptr = ptr[HIGH];
+
+ /* As either the low or high pointer could
+ * consume their partition and move onto the
+ * next chunk, we need to account for both.
+ * In the case where nothing has been allocated
+ * from the high pointer the whole chunk could
+ * be backed unnecessarily - but the granularity
+ * is the chunk size anyway and any non-zero
+ * offset of low pointer from the start of the
+ * chunk would result in the whole chunk being
+ * backed.
+ */
+ read_val = max(high_ptr, low_ptr);
+
/* kbase_check_alloc_sizes() already satisfies
* this, but here to avoid future maintenance
* hazards
*/
WARN_ON(!is_power_of_2(extent_bytes));
-
addr_end = ALIGN(read_val, extent_bytes);
+ } else {
+ addr_end = read_val = READ_ONCE(*ptr);
}
- used_pages = PFN_UP(addr_end) - reg->start_pfn;
+
+ if (addr_end >= (reg->start_pfn << PAGE_SHIFT))
+ used_pages = PFN_UP(addr_end) - reg->start_pfn;
+ else
+ used_pages = reg->used_pages;
}
trace_mali_jit_report(katom, reg, idx, read_val, used_pages);
@@ -877,6 +911,8 @@ static bool jd_submit_atom(struct kbase_context *const kctx,
katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED;
katom->softjob_data = NULL;
+ trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number);
+
#if MALI_JIT_PRESSURE_LIMIT
/* Older API version atoms might have random values where jit_id now
* lives, but we must maintain backwards compatibility - handle the
@@ -1368,7 +1404,7 @@ void kbase_jd_done_worker(struct work_struct *data)
dev_dbg(kbdev->dev, "Enter atom %p done worker for kctx %p\n",
(void *)katom, (void *)kctx);
- KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
kbase_backend_complete_wq(kbdev, katom);
@@ -1508,7 +1544,7 @@ void kbase_jd_done_worker(struct work_struct *data)
if (context_idle)
kbase_pm_context_idle(kbdev);
- KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
dev_dbg(kbdev->dev, "Leave atom %p done worker for kctx %p\n",
(void *)katom, (void *)kctx);
@@ -1546,7 +1582,7 @@ static void jd_cancel_worker(struct work_struct *data)
jctx = &kctx->jctx;
js_kctx_info = &kctx->jctx.sched_info;
- KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
/* This only gets called on contexts that are scheduled out. Hence, we must
* make sure we don't de-ref the number of running jobs (there aren't
@@ -1608,7 +1644,7 @@ void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
- KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0);
kbase_job_check_leave_disjoint(kbdev, katom);
@@ -1640,7 +1676,7 @@ void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
KBASE_DEBUG_ASSERT(NULL != kctx);
dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom);
- KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
/* This should only be done from a context that is not scheduled */
KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));
@@ -1664,7 +1700,7 @@ void kbase_jd_zap_context(struct kbase_context *kctx)
kbdev = kctx->kbdev;
- KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
kbase_js_zap_context(kctx);
diff --git a/mali_kbase/mali_kbase_js.c b/mali_kbase/mali_kbase_js.c
index b3ae604..0b0c5bf 100644
--- a/mali_kbase/mali_kbase_js.c
+++ b/mali_kbase/mali_kbase_js.c
@@ -28,6 +28,7 @@
#include <mali_kbase.h>
#include <mali_kbase_js.h>
#include <tl/mali_kbase_tracepoints.h>
+#include <mali_linux_trace.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_ctx_sched.h>
@@ -37,7 +38,6 @@
#include "mali_kbase_jm.h"
#include "mali_kbase_hwaccess_jm.h"
-
/*
* Private types
*/
@@ -81,22 +81,19 @@ static int kbase_js_get_slot(struct kbase_device *kbdev,
static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
kbasep_js_ctx_job_cb callback);
-/* Helper for trace subcodes */
-#if KBASE_TRACE_ENABLE
-static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
- struct kbase_context *kctx)
+/* Helper for ktrace */
+#if KBASE_KTRACE_ENABLE
+static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
{
return atomic_read(&kctx->refcount);
}
-#else /* KBASE_TRACE_ENABLE */
-static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
- struct kbase_context *kctx)
+#else /* KBASE_KTRACE_ENABLE */
+static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
{
- CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
return 0;
}
-#endif /* KBASE_TRACE_ENABLE */
+#endif /* KBASE_KTRACE_ENABLE */
/*
* Private functions
@@ -187,7 +184,7 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js)
lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
- for (prio = BASE_JD_PRIO_MEDIUM;
+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH;
prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
if (!jsctx_rb_none_to_pull_prio(kctx, js, prio))
return false;
@@ -280,7 +277,7 @@ jsctx_queue_foreach(struct kbase_context *kctx, int js,
{
int prio;
- for (prio = BASE_JD_PRIO_MEDIUM;
+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH;
prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++)
jsctx_queue_foreach_prio(kctx, js, prio, callback);
}
@@ -336,7 +333,7 @@ jsctx_rb_peek(struct kbase_context *kctx, int js)
lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
- for (prio = BASE_JD_PRIO_MEDIUM;
+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH;
prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
struct kbase_jd_atom *katom;
@@ -538,7 +535,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev)
mutex_init(&jsdd->runpool_mutex);
mutex_init(&jsdd->queue_mutex);
- spin_lock_init(&kbdev->hwaccess_lock);
sema_init(&jsdd->schedule_sem, 1);
for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) {
@@ -1185,7 +1181,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx)
/* Determine the new priority for context, as per the priority
* of currently in-use atoms.
*/
- for (prio = BASE_JD_PRIO_MEDIUM;
+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH;
prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
if (kctx->atoms_count[prio]) {
new_priority = prio;
@@ -1217,8 +1213,9 @@ static int js_add_start_rp(struct kbase_jd_atom *const start_katom)
if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS)
return -EINVAL;
- if (start_katom->renderpass_id >= ARRAY_SIZE(kctx->jctx.renderpasses))
- return -EINVAL;
+ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[start_katom->renderpass_id];
@@ -1262,8 +1259,9 @@ static int js_add_end_rp(struct kbase_jd_atom *const end_katom)
if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS)
return -EINVAL;
- if (end_katom->renderpass_id >= ARRAY_SIZE(kctx->jctx.renderpasses))
- return -EINVAL;
+ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[end_katom->renderpass_id];
@@ -1376,8 +1374,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx,
enqueue_required = kbase_js_dep_resolved_submit(kctx, atom);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc,
- kbasep_js_trace_get_refcnt(kbdev, kctx));
+ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc,
+ kbase_ktrace_get_ctx_refcnt(kctx));
/* Context Attribute Refcounting */
kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom);
@@ -1443,8 +1441,8 @@ void kbasep_js_remove_job(struct kbase_device *kbdev,
js_kctx_info = &kctx->jctx.sched_info;
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc,
- kbasep_js_trace_get_refcnt(kbdev, kctx));
+ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc,
+ kbase_ktrace_get_ctx_refcnt(kctx));
/* De-refcount ctx.nr_jobs */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0);
@@ -1530,7 +1528,7 @@ static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release(
* run more jobs than before */
result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL;
- KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB,
+ KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB,
kctx, NULL, 0u, 0);
}
return result;
@@ -1608,10 +1606,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom(
kbdev, kctx, katom_retained_state);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u,
- new_ref_count);
-
if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) &&
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ !kbase_pm_is_gpu_lost(kbdev) &&
+#endif
!kbase_pm_is_suspending(kbdev)) {
/* Context is kept scheduled into an address space even when
* there are no jobs, in this case we have to handle the
@@ -1629,7 +1627,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
* which was previously acquired by kbasep_js_schedule_ctx(). */
if (new_ref_count == 1 &&
(!kbasep_js_is_submit_allowed(js_devdata, kctx) ||
- kbdev->pm.suspending)) {
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbase_pm_is_gpu_lost(kbdev) ||
+#endif
+ kbase_pm_is_suspending(kbdev))) {
int num_slots = kbdev->gpu_props.num_job_slots;
int slot;
@@ -1900,9 +1901,9 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
return false;
}
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL,
+ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL,
0u,
- kbasep_js_trace_get_refcnt(kbdev, kctx));
+ kbase_ktrace_get_ctx_refcnt(kctx));
kbase_ctx_flag_set(kctx, KCTX_SCHEDULED);
@@ -1939,11 +1940,15 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
* kbasep_js_suspend() code will cleanup this context instead (by virtue
* of it being called strictly after the suspend flag is set, and will
* wait for this lock to drop) */
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
+#else
if (kbase_pm_is_suspending(kbdev)) {
+#endif
/* Cause it to leave at some later point */
bool retained;
- retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ retained = kbase_ctx_sched_inc_refcount_nolock(kctx);
KBASE_DEBUG_ASSERT(retained);
kbasep_js_clear_submit_allowed(js_devdata, kctx);
@@ -2016,9 +2021,30 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev,
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
- /* This must never be attempted whilst suspending - i.e. it should only
- * happen in response to a syscall from a user-space thread */
- BUG_ON(kbase_pm_is_suspending(kbdev));
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ /* This should only happen in response to a system call
+ * from a user-space thread.
+ * In a non-arbitrated environment this can never happen
+ * whilst suspending.
+ *
+ * In an arbitrated environment, user-space threads can run
+ * while we are suspended (for example GPU not available
+ * to this VM), however in that case we will block on
+ * the wait event for KCTX_SCHEDULED, since no context
+ * can be scheduled until we have the GPU again.
+ */
+ if (kbdev->arb.arb_if == NULL)
+ if (WARN_ON(kbase_pm_is_suspending(kbdev)))
+ return;
+#else
+ /* This should only happen in response to a system call
+ * from a user-space thread.
+ * In a non-arbitrated environment this can never happen
+ * whilst suspending.
+ */
+ if (WARN_ON(kbase_pm_is_suspending(kbdev)))
+ return;
+#endif
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
@@ -2046,7 +2072,7 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev,
} else {
/* Already scheduled in - We need to retain it to keep the
* corresponding address space */
- WARN_ON(!kbasep_js_runpool_retain_ctx(kbdev, kctx));
+ WARN_ON(!kbase_ctx_sched_inc_refcount(kctx));
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
}
@@ -2136,11 +2162,12 @@ void kbasep_js_resume(struct kbase_device *kbdev)
mutex_lock(&js_devdata->queue_mutex);
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
- for (prio = BASE_JD_PRIO_MEDIUM;
+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH;
prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
struct kbase_context *kctx, *n;
unsigned long flags;
+#ifndef CONFIG_MALI_ARBITER_SUPPORT
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
list_for_each_entry_safe(kctx, n,
@@ -2164,9 +2191,13 @@ void kbasep_js_resume(struct kbase_device *kbdev)
timer_sync =
kbase_js_ctx_list_add_pullable_nolock(
kbdev, kctx, js);
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock,
+ flags);
+
if (timer_sync)
kbase_backend_ctx_count_changed(kbdev);
+
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
@@ -2174,6 +2205,30 @@ void kbasep_js_resume(struct kbase_device *kbdev)
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+#else
+ bool timer_sync = false;
+
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+
+ list_for_each_entry_safe(kctx, n,
+ &kbdev->js_data.ctx_list_unpullable[js][prio],
+ jctx.sched_info.ctx.ctx_list_entry[js]) {
+
+ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) &&
+ kbase_js_ctx_pullable(kctx, js, false))
+ timer_sync |=
+ kbase_js_ctx_list_add_pullable_nolock(
+ kbdev, kctx, js);
+ }
+
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+ if (timer_sync) {
+ mutex_lock(&js_devdata->runpool_mutex);
+ kbase_backend_ctx_count_changed(kbdev);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ }
+#endif
}
}
mutex_unlock(&js_devdata->queue_mutex);
@@ -2354,6 +2409,8 @@ static void kbase_js_evict_deps(struct kbase_context *kctx,
KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) {
/* Remove dependency.*/
x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
+ trace_sysgraph(SGR_DEP_RES, kctx->id,
+ kbase_jd_atom_id(kctx, x_dep));
dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %p\n",
(void *)x_dep);
@@ -2388,7 +2445,11 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
(void *)kctx);
return NULL;
}
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev))
+#else
if (kbase_pm_is_suspending(kbdev))
+#endif
return NULL;
katom = jsctx_rb_peek(kctx, js);
@@ -2451,7 +2512,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++;
jsctx_rb_pull(kctx, katom);
- kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ kbase_ctx_sched_retain_ctx_refcount(kctx);
katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF;
@@ -2490,9 +2551,9 @@ static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom)
if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS)))
return;
- if (WARN_ON(start_katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return;
+ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[start_katom->renderpass_id];
@@ -2599,9 +2660,9 @@ static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom)
if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS)))
return;
- if (WARN_ON(end_katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return;
+ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[end_katom->renderpass_id];
@@ -2856,9 +2917,9 @@ static bool js_complete_start_rp(struct kbase_context *kctx,
if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS)))
return false;
- if (WARN_ON(start_katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return false;
+ compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[start_katom->renderpass_id];
@@ -2952,9 +3013,9 @@ static void js_complete_end_rp(struct kbase_context *kctx,
if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS)))
return;
- if (WARN_ON(end_katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return;
+ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[end_katom->renderpass_id];
@@ -3129,9 +3190,9 @@ static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom)
if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS)))
return true;
- if (WARN_ON(end_katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return true;
+ compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[end_katom->renderpass_id];
@@ -3194,6 +3255,9 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom,
KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL,
katom->slot_nr, 0, TL_JS_EVENT_STOP);
+ trace_sysgraph_gpu(SGR_COMPLETE, kctx->id,
+ kbase_jd_atom_id(katom->kctx, katom), katom->slot_nr);
+
kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0);
/* Unblock cross dependency if present */
@@ -3203,7 +3267,8 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom,
bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr,
false);
x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
-
+ trace_sysgraph(SGR_DEP_RES, kctx->id,
+ kbase_jd_atom_id(katom->kctx, x_dep));
dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %p\n",
(void *)x_dep);
@@ -3263,9 +3328,9 @@ bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom)
return true;
}
- if (WARN_ON(katom->renderpass_id >=
- ARRAY_SIZE(kctx->jctx.renderpasses)))
- return true;
+ compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
+ ARRAY_SIZE(kctx->jctx.renderpasses),
+ "Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
/* We can read a subset of renderpass state without holding
@@ -3399,11 +3464,12 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask)
kbase_ctx_flag_clear(kctx, KCTX_PULLED);
- if (!kbase_jm_kick(kbdev, 1 << js))
+ if (!kbase_jm_kick(kbdev, 1 << js)) {
dev_dbg(kbdev->dev,
"No more jobs can be submitted (s:%d)\n",
js);
js_mask &= ~(1 << js);
+ }
if (!kbase_ctx_flag(kctx, KCTX_PULLED)) {
bool pullable;
@@ -3582,8 +3648,7 @@ void kbase_js_zap_context(struct kbase_context *kctx)
* back (this already cancels the jobs)
*/
- KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u,
- kbase_ctx_flag(kctx, KCTX_SCHEDULED));
+ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED));
dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx);
@@ -3603,8 +3668,7 @@ void kbase_js_zap_context(struct kbase_context *kctx)
/* Case c: didn't evict, but it is scheduled - it's in the Run
* Pool */
- KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u,
- kbase_ctx_flag(kctx, KCTX_SCHEDULED));
+ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED));
dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx);
/* Disable the ctx from submitting any more jobs */
@@ -3615,7 +3679,7 @@ void kbase_js_zap_context(struct kbase_context *kctx)
/* Retain and (later) release the context whilst it is is now
* disallowed from submitting jobs - ensures that someone
* somewhere will be removing the context later on */
- was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx);
/* Since it's scheduled and we have the jsctx_mutex, it must be
* retained successfully */
@@ -3639,7 +3703,7 @@ void kbase_js_zap_context(struct kbase_context *kctx)
kbasep_js_runpool_release_ctx(kbdev, kctx);
}
- KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
+ KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
/* After this, you must wait on both the
* kbase_jd_context::zero_jobs_wait and the
@@ -3685,7 +3749,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL,
+ KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL,
0u, trace_get_refcnt(kbdev, kctx));
/* Invoke callback on jobs on each slot in turn */
@@ -3694,81 +3758,3 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
-
-
-/* Hold the mmu_hw_mutex and hwaccess_lock for this */
-bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
- struct kbase_context *kctx)
-{
- bool result = false;
- int as_nr;
-
- if (WARN_ON(kbdev == NULL))
- return result;
-
- if (WARN_ON(kctx == NULL))
- return result;
-
- lockdep_assert_held(&kbdev->hwaccess_lock);
-
- as_nr = kctx->as_nr;
- if (atomic_read(&kctx->refcount) > 0) {
- KBASE_DEBUG_ASSERT(as_nr >= 0);
-
- kbase_ctx_sched_retain_ctx_refcount(kctx);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx,
- NULL, 0u, atomic_read(&kctx->refcount));
- result = true;
- }
-
- return result;
-}
-
-
-bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev,
- struct kbase_context *kctx)
-{
- unsigned long flags;
- bool result = false;
-
- if (WARN_ON(kbdev == NULL))
- return result;
-
- if (WARN_ON(kctx == NULL))
- return result;
-
- mutex_lock(&kbdev->mmu_hw_mutex);
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
- result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
- mutex_unlock(&kbdev->mmu_hw_mutex);
-
- return result;
-}
-
-struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev,
- int as_nr)
-{
- unsigned long flags;
- struct kbase_context *found_kctx = NULL;
-
- if (WARN_ON(kbdev == NULL))
- return NULL;
-
- if (WARN_ON(as_nr < 0))
- return NULL;
-
- if (WARN_ON(as_nr >= BASE_MAX_NR_AS))
- return NULL;
-
- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
-
- found_kctx = kbdev->as_to_kctx[as_nr];
-
- if (found_kctx != NULL)
- kbase_ctx_sched_retain_ctx_refcount(found_kctx);
-
- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-
- return found_kctx;
-}
diff --git a/mali_kbase/mali_kbase_js.h b/mali_kbase/mali_kbase_js.h
index 51ab023..541acd4 100644
--- a/mali_kbase/mali_kbase_js.h
+++ b/mali_kbase/mali_kbase_js.h
@@ -30,100 +30,11 @@
#ifndef _KBASE_JS_H_
#define _KBASE_JS_H_
-#include "mali_kbase_js_defs.h"
#include "context/mali_kbase_context.h"
#include "mali_kbase_defs.h"
#include "mali_kbase_debug.h"
#include <mali_kbase_ctx_sched.h>
-
-#include <jm/mali_kbase_jm_js.h>
-
-/**
- * kbasep_js_runpool_retain_ctx_nolock - Refcount a context as being busy,
- * preventing it from being scheduled
- * out.
- *
- * This function can safely be called from IRQ context.
- *
- * The following locks must be held by the caller:
- * * mmu_hw_mutex, hwaccess_lock
- *
- * Return: value true if the retain succeeded, and the context will not be
- * scheduled out, otherwise false if the retain failed (because the context
- * is being/has been scheduled out).
- */
-bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
- struct kbase_context *kctx);
-
-/**
- * kbasep_js_runpool_retain_ctx - Refcount a context as being busy, preventing
- * it from being scheduled out.
- *
- * This function can safely be called from IRQ context.
- *
- * The following locking conditions are made on the caller:
- * * it must not hold mmu_hw_mutex and hwaccess_lock, because they will be
- * used internally.
- *
- * Return: value true if the retain succeeded, and the context will not be
- * scheduled out, otherwise false if the retain failed (because the context
- * is being/has been scheduled out).
- */
-bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev,
- struct kbase_context *kctx);
-
-/**
- * kbasep_js_runpool_lookup_ctx - Lookup a context in the Run Pool based upon
- * its current address space and ensure that
- * is stays scheduled in.
- *
- * The context is refcounted as being busy to prevent it from scheduling
- * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
- * longer required to stay scheduled in.
- *
- * This function can safely be called from IRQ context.
- *
- * The following locking conditions are made on the caller:
- * * it must not hold the hwaccess_lock, because it will be used internally.
- * If the hwaccess_lock is already held, then the caller should use
- * kbasep_js_runpool_lookup_ctx_nolock() instead.
- *
- * Return: a valid struct kbase_context on success, which has been refcounted
- * as being busy or return NULL on failure, indicating that no context was found
- * in as_nr.
- */
-struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev,
- int as_nr);
-
-/**
- * kbasep_js_runpool_lookup_ctx_noretain - Variant of
- * kbasep_js_runpool_lookup_ctx() that can be used when the
- * context is guaranteed to be already previously retained.
- *
- * It is a programming error to supply the as_nr of a context that has not
- * been previously retained/has a busy refcount of zero. The only exception is
- * when there is no ctx in as_nr (NULL returned).
- *
- * The following locking conditions are made on the caller:
- * * it must not hold the hwaccess_lock, because it will be used internally.
- *
- * Return: a valid struct kbase_context on success, with a refcount that is
- * guaranteed to be non-zero and unmodified by this function or
- * return NULL on failure, indicating that no context was found in as_nr.
- */
-static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(
- struct kbase_device *kbdev, int as_nr)
-{
- struct kbase_context *found_kctx;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
-
- found_kctx = kbdev->as_to_kctx[as_nr];
- KBASE_DEBUG_ASSERT(found_kctx == NULL ||
- atomic_read(&found_kctx->refcount) > 0);
-
- return found_kctx;
-}
+#include "jm/mali_kbase_jm_js.h"
+#include "jm/mali_kbase_js_defs.h"
#endif /* _KBASE_JS_H_ */
diff --git a/mali_kbase/mali_kbase_js_ctx_attr.c b/mali_kbase/mali_kbase_js_ctx_attr.c
index 1ff230c..141d04a 100644
--- a/mali_kbase/mali_kbase_js_ctx_attr.c
+++ b/mali_kbase/mali_kbase_js_ctx_attr.c
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -67,7 +67,7 @@ static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, s
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) {
/* First refcount indicates a state change */
runpool_state_changed = true;
- KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute);
+ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute);
}
}
@@ -112,7 +112,7 @@ static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev,
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) {
/* Last de-refcount indicates a state change */
runpool_state_changed = true;
- KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
+ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
}
}
@@ -149,7 +149,7 @@ static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struc
if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
/* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */
- KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
+ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute);
}
@@ -185,7 +185,7 @@ static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, stru
lockdep_assert_held(&kbdev->hwaccess_lock);
/* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */
runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute);
- KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute);
+ KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute);
}
/* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */
diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c
index 2362e22..4a1004b 100644
--- a/mali_kbase/mali_kbase_mem.c
+++ b/mali_kbase/mali_kbase_mem.c
@@ -2983,6 +2983,63 @@ static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data)
KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops,
kbase_jit_debugfs_phys_get);
+#if MALI_JIT_PRESSURE_LIMIT
+static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data)
+{
+ struct kbase_context *kctx = data->kctx;
+ struct kbase_va_region *reg;
+
+ mutex_lock(&kctx->jctx.lock);
+ mutex_lock(&kctx->jit_evict_lock);
+ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
+ data->active_value += reg->used_pages;
+ }
+ mutex_unlock(&kctx->jit_evict_lock);
+ mutex_unlock(&kctx->jctx.lock);
+
+ return 0;
+}
+
+KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_used_fops,
+ kbase_jit_debugfs_used_get);
+
+static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx,
+ struct kbase_va_region *reg, size_t pages_needed,
+ size_t *freed, bool shrink);
+
+static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data)
+{
+ struct kbase_context *kctx = data->kctx;
+ struct kbase_va_region *reg;
+
+ mutex_lock(&kctx->jctx.lock);
+ kbase_gpu_vm_lock(kctx);
+ mutex_lock(&kctx->jit_evict_lock);
+ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
+ int err;
+ size_t freed = 0u;
+
+ err = kbase_mem_jit_trim_pages_from_region(kctx, reg,
+ SIZE_MAX, &freed, false);
+
+ if (err) {
+ /* Failed to calculate, try the next region */
+ continue;
+ }
+
+ data->active_value += freed;
+ }
+ mutex_unlock(&kctx->jit_evict_lock);
+ kbase_gpu_vm_unlock(kctx);
+ mutex_unlock(&kctx->jctx.lock);
+
+ return 0;
+}
+
+KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_trim_fops,
+ kbase_jit_debugfs_trim_get);
+#endif /* MALI_JIT_PRESSURE_LIMIT */
+
void kbase_jit_debugfs_init(struct kbase_context *kctx)
{
/* prevent unprivileged use of debug file system
@@ -3021,6 +3078,22 @@ void kbase_jit_debugfs_init(struct kbase_context *kctx)
*/
debugfs_create_file("mem_jit_phys", mode, kctx->kctx_dentry,
kctx, &kbase_jit_debugfs_phys_fops);
+#if MALI_JIT_PRESSURE_LIMIT
+ /*
+ * Debugfs entry for getting the number of pages used
+ * by JIT allocations for estimating the physical pressure
+ * limit.
+ */
+ debugfs_create_file("mem_jit_used", mode, kctx->kctx_dentry,
+ kctx, &kbase_jit_debugfs_used_fops);
+
+ /*
+ * Debugfs entry for getting the number of pages that could
+ * be trimmed to free space for more JIT allocations.
+ */
+ debugfs_create_file("mem_jit_trim", mode, kctx->kctx_dentry,
+ kctx, &kbase_jit_debugfs_trim_fops);
+#endif /* MALI_JIT_PRESSURE_LIMIT */
}
#endif /* CONFIG_DEBUG_FS */
@@ -3065,8 +3138,8 @@ int kbase_jit_init(struct kbase_context *kctx)
INIT_LIST_HEAD(&kctx->jit_destroy_head);
INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker);
- INIT_LIST_HEAD(&kctx->jit_pending_alloc);
- INIT_LIST_HEAD(&kctx->jit_atoms_head);
+ INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head);
+ INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc);
mutex_unlock(&kctx->jit_evict_lock);
kctx->jit_max_allocations = 0;
@@ -3081,7 +3154,7 @@ int kbase_jit_init(struct kbase_context *kctx)
* the alignment requirements.
*/
static bool meet_size_and_tiler_align_top_requirements(struct kbase_context *kctx,
- struct kbase_va_region *walker, struct base_jit_alloc_info *info)
+ struct kbase_va_region *walker, const struct base_jit_alloc_info *info)
{
bool meet_reqs = true;
@@ -3103,7 +3176,7 @@ static bool meet_size_and_tiler_align_top_requirements(struct kbase_context *kct
*/
static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx,
struct kbase_va_region *reg, size_t pages_needed,
- size_t *freed)
+ size_t *freed, bool shrink)
{
int err = 0;
size_t available_pages = 0u;
@@ -3199,10 +3272,11 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx,
available_pages = old_pages - reg->used_pages;
to_free = min(available_pages, pages_needed);
- new_pages -= to_free;
-
- err = kbase_mem_shrink(kctx, reg, new_pages);
+ if (shrink) {
+ new_pages -= to_free;
+ err = kbase_mem_shrink(kctx, reg, new_pages);
+ }
out:
trace_mali_jit_trim_from_region(reg, to_free, old_pages,
available_pages, new_pages);
@@ -3210,7 +3284,25 @@ out:
return err;
}
-size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
+
+/**
+ * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been
+ * freed
+ * @kctx: Pointer to the kbase context whose active JIT allocations will be
+ * checked.
+ * @pages_needed: The maximum number of pages to trim.
+ *
+ * This functions checks all active JIT allocations in @kctx for unused pages
+ * at the end, and trim the backed memory regions of those allocations down to
+ * the used portion and free the unused pages into the page pool.
+ *
+ * Specifying @pages_needed allows us to stop early when there's enough
+ * physical memory freed to sufficiently bring down the total JIT physical page
+ * usage (e.g. to below the pressure limit)
+ *
+ * Return: Total number of successfully freed pages
+ */
+static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
size_t pages_needed)
{
struct kbase_va_region *reg, *tmp;
@@ -3223,7 +3315,7 @@ size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
size_t freed = 0u;
err = kbase_mem_jit_trim_pages_from_region(kctx, reg,
- pages_needed, &freed);
+ pages_needed, &freed, true);
if (err) {
/* Failed to trim, try the next region */
@@ -3246,7 +3338,8 @@ size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
#endif /* MALI_JIT_PRESSURE_LIMIT */
static int kbase_jit_grow(struct kbase_context *kctx,
- struct base_jit_alloc_info *info, struct kbase_va_region *reg)
+ const struct base_jit_alloc_info *info,
+ struct kbase_va_region *reg)
{
size_t delta;
size_t pages_required;
@@ -3399,30 +3492,143 @@ static void trace_jit_stats(struct kbase_context *kctx,
max_allocations, alloc_count, va_pages, ph_pages);
}
-struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
- struct base_jit_alloc_info *info)
+#if MALI_JIT_PRESSURE_LIMIT
+/**
+ * get_jit_backed_pressure() - calculate the physical backing of all JIT
+ * allocations
+ *
+ * @kctx: Pointer to the kbase context whose active JIT allocations will be
+ * checked
+ *
+ * Return: number of pages that are committed by JIT allocations
+ */
+static size_t get_jit_backed_pressure(struct kbase_context *kctx)
{
- struct kbase_va_region *reg = NULL;
+ size_t backed_pressure = 0;
+ int jit_id;
+
+ lockdep_assert_held(&kctx->jctx.lock);
+
+ kbase_gpu_vm_lock(kctx);
+ for (jit_id = 0; jit_id <= BASE_JIT_ALLOC_COUNT; jit_id++) {
+ struct kbase_va_region *reg = kctx->jit_alloc[jit_id];
+
+ if (reg && (reg != KBASE_RESERVED_REG_JIT_ALLOC)) {
+ /* If region has no report, be pessimistic */
+ if (reg->used_pages == reg->nr_pages) {
+ backed_pressure += reg->nr_pages;
+ } else {
+ backed_pressure +=
+ kbase_reg_current_backed_size(reg);
+ }
+ }
+ }
+ kbase_gpu_vm_unlock(kctx);
+
+ return backed_pressure;
+}
+
+/**
+ * jit_trim_necessary_pages() - calculate and trim the least pages possible to
+ * satisfy a new JIT allocation
+ *
+ * @kctx: Pointer to the kbase context
+ * @info: Pointer to JIT allocation information for the new allocation
+ *
+ * Before allocating a new just-in-time memory region or reusing a previous
+ * one, ensure that the total JIT physical page usage also will not exceed the
+ * pressure limit.
+ *
+ * If there are no reported-on allocations, then we already guarantee this will
+ * be the case - because our current pressure then only comes from the va_pages
+ * of each JIT region, hence JIT physical page usage is guaranteed to be
+ * bounded by this.
+ *
+ * However as soon as JIT allocations become "reported on", the pressure is
+ * lowered to allow new JIT regions to be allocated. It is after such a point
+ * that the total JIT physical page usage could (either now or in the future on
+ * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly
+ * allocated JIT regions. Hence, trim any "reported on" regions.
+ *
+ * Any pages freed will go into the pool and be allocated from there in
+ * kbase_mem_alloc().
+ */
+static void jit_trim_necessary_pages(struct kbase_context *kctx,
+ const struct base_jit_alloc_info *info)
+{
+ size_t backed_pressure = 0;
+ size_t needed_pages = 0;
+
+ backed_pressure = get_jit_backed_pressure(kctx);
+
+ /* It is possible that this is the case - if this is the first
+ * allocation after "ignore_pressure_limit" allocation.
+ */
+ if (backed_pressure > kctx->jit_phys_pages_limit) {
+ needed_pages +=
+ (backed_pressure - kctx->jit_phys_pages_limit)
+ + info->va_pages;
+ } else {
+ size_t backed_diff =
+ kctx->jit_phys_pages_limit - backed_pressure;
+
+ if (info->va_pages > backed_diff)
+ needed_pages += info->va_pages - backed_diff;
+ }
+
+ if (needed_pages) {
+ size_t trimmed_pages = kbase_mem_jit_trim_pages(kctx,
+ needed_pages);
+
+ /* This should never happen - we already asserted that
+ * we are not violating JIT pressure limit in earlier
+ * checks, which means that in-flight JIT allocations
+ * must have enough unused pages to satisfy the new
+ * allocation
+ */
+ WARN_ON(trimmed_pages < needed_pages);
+ }
+}
+#endif /* MALI_JIT_PRESSURE_LIMIT */
+
+/**
+ * jit_allow_allocate() - check whether basic conditions are satisfied to allow
+ * a new JIT allocation
+ *
+ * @kctx: Pointer to the kbase context
+ * @info: Pointer to JIT allocation information for the new allocation
+ * @ignore_pressure_limit: Flag to indicate whether JIT pressure limit check
+ * should be ignored
+ *
+ * Return: true if allocation can be executed, false otherwise
+ */
+static bool jit_allow_allocate(struct kbase_context *kctx,
+ const struct base_jit_alloc_info *info,
+ bool ignore_pressure_limit)
+{
+ lockdep_assert_held(&kctx->jctx.lock);
#if MALI_JIT_PRESSURE_LIMIT
- if (info->va_pages > (kctx->jit_phys_pages_limit -
- kctx->jit_current_phys_pressure) &&
- kctx->jit_current_phys_pressure > 0) {
+ if (likely(!ignore_pressure_limit) &&
+ ((kctx->jit_phys_pages_limit <= kctx->jit_current_phys_pressure) ||
+ (info->va_pages > (kctx->jit_phys_pages_limit - kctx->jit_current_phys_pressure)))) {
dev_dbg(kctx->kbdev->dev,
"Max JIT page allocations limit reached: active pages %llu, max pages %llu\n",
kctx->jit_current_phys_pressure + info->va_pages,
kctx->jit_phys_pages_limit);
- return NULL;
+ return false;
}
#endif /* MALI_JIT_PRESSURE_LIMIT */
+
if (kctx->jit_current_allocations >= kctx->jit_max_allocations) {
/* Too many current allocations */
dev_dbg(kctx->kbdev->dev,
"Max JIT allocations limit reached: active allocations %d, max allocations %d\n",
kctx->jit_current_allocations,
kctx->jit_max_allocations);
- return NULL;
+ return false;
}
+
if (info->max_allocations > 0 &&
kctx->jit_current_allocations_per_bin[info->bin_id] >=
info->max_allocations) {
@@ -3432,34 +3638,26 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
info->bin_id,
kctx->jit_current_allocations_per_bin[info->bin_id],
info->max_allocations);
- return NULL;
+ return false;
}
+ return true;
+}
+
+struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
+ const struct base_jit_alloc_info *info,
+ bool ignore_pressure_limit)
+{
+ struct kbase_va_region *reg = NULL;
+
+ lockdep_assert_held(&kctx->jctx.lock);
+
+ if (!jit_allow_allocate(kctx, info, ignore_pressure_limit))
+ return NULL;
+
#if MALI_JIT_PRESSURE_LIMIT
- /* Before allocating a new just-in-time memory region or reusing a
- * previous one, ensure that the total JIT physical page usage also will
- * not exceed the pressure limit.
- *
- * If there are no reported-on allocations, then we already guarantee
- * this will be the case - because our current pressure then only comes
- * from the va_pages of each JIT region, hence JIT physical page usage
- * is guaranteed to be bounded by this.
- *
- * However as soon as JIT allocations become "reported on", the
- * pressure is lowered to allow new JIT regions to be allocated. It is
- * after such a point that the total JIT physical page usage could
- * (either now or in the future on a grow-on-GPU-page-fault) exceed the
- * pressure limit, but only on newly allocated JIT regions. Hence, trim
- * any "reported on" regions.
- *
- * Any pages freed will go into the pool and be allocated from there in
- * kbase_mem_alloc().
- *
- * In future, GPUCORE-21217: Only do this when physical page usage
- * could exceed the pressure limit, and only trim as much as is
- * necessary.
- */
- kbase_mem_jit_trim_pages(kctx, SIZE_MAX);
+ if (!ignore_pressure_limit)
+ jit_trim_necessary_pages(kctx, info);
#endif /* MALI_JIT_PRESSURE_LIMIT */
mutex_lock(&kctx->jit_evict_lock);
@@ -3572,7 +3770,10 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
dev_dbg(kctx->kbdev->dev,
"JIT allocation resize failed: va_pages 0x%llx, commit_pages 0x%llx\n",
info->va_pages, info->commit_pages);
- goto update_failed_unlocked;
+ mutex_lock(&kctx->jit_evict_lock);
+ list_move(&reg->jit_node, &kctx->jit_pool_head);
+ mutex_unlock(&kctx->jit_evict_lock);
+ return NULL;
}
} else {
/* No suitable JIT allocation was found so create a new one */
@@ -3598,7 +3799,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
dev_dbg(kctx->kbdev->dev,
"Failed to allocate JIT memory: va_pages 0x%llx, commit_pages 0x%llx\n",
info->va_pages, info->commit_pages);
- goto out_unlocked;
+ return NULL;
}
mutex_lock(&kctx->jit_evict_lock);
@@ -3624,13 +3825,6 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
#endif /* MALI_JIT_PRESSURE_LIMIT */
return reg;
-
-update_failed_unlocked:
- mutex_lock(&kctx->jit_evict_lock);
- list_move(&reg->jit_node, &kctx->jit_pool_head);
- mutex_unlock(&kctx->jit_evict_lock);
-out_unlocked:
- return NULL;
}
void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg)
@@ -3848,35 +4042,6 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx,
kctx->jit_current_phys_pressure -= diff;
reg->used_pages = new_used_pages;
-
- /* In the case of pressure reduced on a free, don't attempt to
- * trim the region: it will soon be placed on the evict_list
- * so that if we really were close to running out of memory then
- * the shrinker can reclaim the memory.
- */
- if ((flags & KBASE_JIT_REPORT_ON_ALLOC_OR_FREE) == 0u) {
- size_t freed;
- int err;
-
- kbase_gpu_vm_lock(kctx);
- /* If this was from an allocation that a single
- * BASE_JD_REQ_SOFT_JIT_ALLOC atom that is allowed to
- * breach the pressure limit, then check whether we can
- * bring the total JIT physical page below (or at least
- * nearer) the pressure limit.
- *
- * In future, GPUCORE-21217: Only do this when physical
- * page usage currently exceeds the pressure limit, and
- * only trim as much as is necessary.
- */
- err = kbase_mem_jit_trim_pages_from_region(kctx, reg,
- SIZE_MAX, &freed);
- kbase_gpu_vm_unlock(kctx);
-
- CSTD_UNUSED(freed);
- /* Nothing to do if trimming failed */
- CSTD_UNUSED(err);
- }
} else {
/* We increased the number of used pages */
diff = new_used_pages - reg->used_pages;
diff --git a/mali_kbase/mali_kbase_mem.h b/mali_kbase/mali_kbase_mem.h
index 3f74492..6e921ec 100644
--- a/mali_kbase/mali_kbase_mem.h
+++ b/mali_kbase/mali_kbase_mem.h
@@ -1457,11 +1457,13 @@ int kbase_jit_init(struct kbase_context *kctx);
* kbase_jit_allocate - Allocate JIT memory
* @kctx: kbase context
* @info: JIT allocation information
+ * @ignore_pressure_limit: Whether the JIT memory pressure limit is ignored
*
* Return: JIT allocation on success or NULL on failure.
*/
struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
- struct base_jit_alloc_info *info);
+ const struct base_jit_alloc_info *info,
+ bool ignore_pressure_limit);
/**
* kbase_jit_free - Free a JIT allocation
@@ -1560,26 +1562,6 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx,
void kbase_jit_report_update_pressure(struct kbase_context *kctx,
struct kbase_va_region *reg, u64 new_used_pages,
unsigned int flags);
-
-/**
- * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been
- * freed
- * @kctx: Pointer to the kbase context whose active JIT allocations will be
- * checked.
- * @pages_needed: The maximum number of pages to trim.
- *
- * This functions checks all active JIT allocations in @kctx for unused pages
- * at the end, and trim the backed memory regions of those allocations down to
- * the used portion and free the unused pages into the page pool.
- *
- * Specifying @pages_needed allows us to stop early when there's enough
- * physical memory freed to sufficiently bring down the total JIT physical page
- * usage (e.g. to below the pressure limit)
- *
- * Return: Total number of successfully freed pages
- */
-size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
- size_t pages_needed);
#endif /* MALI_JIT_PRESSURE_LIMIT */
/**
diff --git a/mali_kbase/mali_kbase_pm.c b/mali_kbase/mali_kbase_pm.c
index 2adbb21..b9ed8c3 100644
--- a/mali_kbase/mali_kbase_pm.c
+++ b/mali_kbase/mali_kbase_pm.c
@@ -33,6 +33,11 @@
#include <mali_kbase_hwcnt_context.h>
#include <mali_kbase_pm.h>
+#include <mali_kbase_pm_internal.h>
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+#include <arbiter/mali_kbase_arbiter_pm.h>
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags)
{
@@ -46,27 +51,36 @@ void kbase_pm_halt(struct kbase_device *kbdev)
void kbase_pm_context_active(struct kbase_device *kbdev)
{
- (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE);
+ (void)kbase_pm_context_active_handle_suspend(kbdev,
+ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE);
}
-int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler)
+int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev,
+ enum kbase_pm_suspend_handler suspend_handler)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
int c;
KBASE_DEBUG_ASSERT(kbdev != NULL);
+ dev_dbg(kbdev->dev, "%s - reason = %d, pid = %d\n", __func__,
+ suspend_handler, current->pid);
+ kbase_pm_lock(kbdev);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler))
+ return 1;
+
+ if (kbase_pm_is_suspending(kbdev) ||
+ kbase_pm_is_gpu_lost(kbdev)) {
+#else
if (kbase_pm_is_suspending(kbdev)) {
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
switch (suspend_handler) {
case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE:
if (kbdev->pm.active_count != 0)
break;
/* FALLTHROUGH */
case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE:
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
return 1;
case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE:
@@ -77,16 +91,20 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbas
}
}
c = ++kbdev->pm.active_count;
- KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c);
+ KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, c);
if (c == 1) {
- /* First context active: Power on the GPU and any cores requested by
- * the policy */
+ /* First context active: Power on the GPU and
+ * any cores requested by the policy
+ */
kbase_hwaccess_pm_gpu_active(kbdev);
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
}
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
+ dev_dbg(kbdev->dev, "%s %d\n", __func__, kbdev->pm.active_count);
return 0;
}
@@ -95,17 +113,15 @@ KBASE_EXPORT_TEST_API(kbase_pm_context_active);
void kbase_pm_context_idle(struct kbase_device *kbdev)
{
- struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
int c;
KBASE_DEBUG_ASSERT(kbdev != NULL);
- mutex_lock(&js_devdata->runpool_mutex);
- mutex_lock(&kbdev->pm.lock);
+ kbase_pm_lock(kbdev);
c = --kbdev->pm.active_count;
- KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c);
+ KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, c);
KBASE_DEBUG_ASSERT(c >= 0);
@@ -113,20 +129,21 @@ void kbase_pm_context_idle(struct kbase_device *kbdev)
/* Last context has gone idle */
kbase_hwaccess_pm_gpu_idle(kbdev);
- /* Wake up anyone waiting for this to become 0 (e.g. suspend). The
- * waiters must synchronize with us by locking the pm.lock after
- * waiting.
+ /* Wake up anyone waiting for this to become 0 (e.g. suspend).
+ * The waiters must synchronize with us by locking the pm.lock
+ * after waiting.
*/
wake_up(&kbdev->pm.zero_active_count_wait);
}
- mutex_unlock(&kbdev->pm.lock);
- mutex_unlock(&js_devdata->runpool_mutex);
+ kbase_pm_unlock(kbdev);
+ dev_dbg(kbdev->dev, "%s %d (pid = %d)\n", __func__,
+ kbdev->pm.active_count, current->pid);
}
KBASE_EXPORT_TEST_API(kbase_pm_context_idle);
-void kbase_pm_suspend(struct kbase_device *kbdev)
+void kbase_pm_driver_suspend(struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev);
@@ -141,13 +158,31 @@ void kbase_pm_suspend(struct kbase_device *kbdev)
kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx);
mutex_lock(&kbdev->pm.lock);
- KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
+ if (WARN_ON(kbase_pm_is_suspending(kbdev))) {
+ mutex_unlock(&kbdev->pm.lock);
+ return;
+ }
kbdev->pm.suspending = true;
mutex_unlock(&kbdev->pm.lock);
- /* From now on, the active count will drop towards zero. Sometimes, it'll
- * go up briefly before going down again. However, once it reaches zero it
- * will stay there - guaranteeing that we've idled all pm references */
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbdev->arb.arb_if) {
+ int i;
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+ kbase_disjoint_state_up(kbdev);
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
+ kbase_job_slot_softstop(kbdev, i, NULL);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+ }
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+
+ /* From now on, the active count will drop towards zero. Sometimes,
+ * it'll go up briefly before going down again. However, once
+ * it reaches zero it will stay there - guaranteeing that we've idled
+ * all pm references
+ */
/* Suspend job scheduler and associated components, so that it releases all
* the PM active count references */
@@ -155,16 +190,29 @@ void kbase_pm_suspend(struct kbase_device *kbdev)
/* Wait for the active count to reach zero. This is not the same as
* waiting for a power down, since not all policies power down when this
- * reaches zero. */
- wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0);
+ * reaches zero.
+ */
+ dev_dbg(kbdev->dev, ">wait_event - waiting for active_count == 0 (pid = %d)\n",
+ current->pid);
+ wait_event(kbdev->pm.zero_active_count_wait,
+ kbdev->pm.active_count == 0);
+ dev_dbg(kbdev->dev, ">wait_event - waiting done\n");
/* NOTE: We synchronize with anything that was just finishing a
- * kbase_pm_context_idle() call by locking the pm.lock below */
-
+ * kbase_pm_context_idle() call by locking the pm.lock below
+ */
kbase_hwaccess_pm_suspend(kbdev);
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbdev->arb.arb_if) {
+ mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock);
+ kbase_arbiter_pm_vm_stopped(kbdev);
+ mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock);
+ }
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
}
-void kbase_pm_resume(struct kbase_device *kbdev)
+void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start)
{
unsigned long flags;
@@ -172,18 +220,28 @@ void kbase_pm_resume(struct kbase_device *kbdev)
kbase_hwaccess_pm_resume(kbdev);
/* Initial active call, to power on the GPU/cores if needed */
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ (void)kbase_pm_context_active_handle_suspend(kbdev,
+ (arb_gpu_start ?
+ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED :
+ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE));
+#else
kbase_pm_context_active(kbdev);
+#endif
/* Resume any blocked atoms (which may cause contexts to be scheduled in
- * and dependent atoms to run) */
+ * and dependent atoms to run)
+ */
kbase_resume_suspended_soft_jobs(kbdev);
/* Resume the Job Scheduler and associated components, and start running
- * atoms */
+ * atoms
+ */
kbasep_js_resume(kbdev);
/* Matching idle call, to power off the GPU/cores if we didn't actually
- * need it and the policy doesn't want it on */
+ * need it and the policy doesn't want it on
+ */
kbase_pm_context_idle(kbdev);
/* Re-enable GPU hardware counters */
@@ -194,3 +252,27 @@ void kbase_pm_resume(struct kbase_device *kbdev)
/* Resume vinstr */
kbase_vinstr_resume(kbdev->vinstr_ctx);
}
+
+void kbase_pm_suspend(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbdev->arb.arb_if)
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT);
+ else
+ kbase_pm_driver_suspend(kbdev);
+#else
+ kbase_pm_driver_suspend(kbdev);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+}
+
+void kbase_pm_resume(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ if (kbdev->arb.arb_if)
+ kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT);
+ else
+ kbase_pm_driver_resume(kbdev, false);
+#else
+ kbase_pm_driver_resume(kbdev, false);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+}
diff --git a/mali_kbase/mali_kbase_pm.h b/mali_kbase/mali_kbase_pm.h
index 59a0314..257f959 100644
--- a/mali_kbase/mali_kbase_pm.h
+++ b/mali_kbase/mali_kbase_pm.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2010-2015,2018 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -40,7 +40,8 @@
*
* Must be called before any other power management function
*
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
+ * @param kbdev The kbase device structure for the device
+ * (must be a valid pointer)
*
* @return 0 if the power management framework was successfully initialized.
*/
@@ -78,8 +79,9 @@ void kbase_pm_term(struct kbase_device *kbdev);
/** Increment the count of active contexts.
*
- * This function should be called when a context is about to submit a job. It informs the active power policy that the
- * GPU is going to be in use shortly and the policy is expected to start turning on the GPU.
+ * This function should be called when a context is about to submit a job.
+ * It informs the active power policy that the GPU is going to be in use shortly
+ * and the policy is expected to start turning on the GPU.
*
* This function will block until the GPU is available.
*
@@ -98,16 +100,24 @@ void kbase_pm_context_active(struct kbase_device *kbdev);
/** Handler codes for doing kbase_pm_context_active_handle_suspend() */
enum kbase_pm_suspend_handler {
/** A suspend is not expected/not possible - this is the same as
- * kbase_pm_context_active() */
+ * kbase_pm_context_active()
+ */
KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE,
/** If we're suspending, fail and don't increase the active count */
KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE,
- /** If we're suspending, succeed and allow the active count to increase iff
- * it didn't go from 0->1 (i.e., we didn't re-activate the GPU).
+ /** If we're suspending, succeed and allow the active count to increase
+ * if it didn't go from 0->1 (i.e., we didn't re-activate the GPU).
*
* This should only be used when there is a bounded time on the activation
- * (e.g. guarantee it's going to be idled very soon after) */
- KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE
+ * (e.g. guarantee it's going to be idled very soon after)
+ */
+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE,
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+ /** Special case when Arbiter has notified we can use GPU.
+ * Active count should always start at 0 in this case.
+ */
+ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED,
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
};
/** Suspend 'safe' variant of kbase_pm_context_active()
@@ -129,8 +139,9 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbas
/** Decrement the reference count of active contexts.
*
- * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power
- * policy so the calling code should ensure that it does not access the GPU's registers.
+ * This function should be called when a context becomes idle.
+ * After this call the GPU may be turned off by the power policy so the calling
+ * code should ensure that it does not access the GPU's registers.
*
* @param kbdev The kbase device structure for the device (must be a valid pointer)
*/
@@ -162,6 +173,8 @@ void kbase_pm_suspend(struct kbase_device *kbdev);
* This is called in response to an OS resume event, and calls into the various
* kbase components to complete the resume.
*
+ * Also called when using VM arbiter, when GPU access has been granted.
+ *
* @param kbdev The kbase device structure for the device (must be a valid pointer)
*/
void kbase_pm_resume(struct kbase_device *kbdev);
@@ -177,4 +190,55 @@ void kbase_pm_resume(struct kbase_device *kbdev);
*/
void kbase_pm_vsync_callback(int buffer_updated, void *data);
-#endif /* _KBASE_PM_H_ */
+/**
+ * kbase_pm_driver_suspend() - Put GPU and driver in suspend state
+ * @param kbdev The kbase device structure for the device
+ * (must be a valid pointer)
+ *
+ * Suspend the GPU and prevent any further register accesses to it from Kernel
+ * threads.
+ *
+ * This is called in response to an OS suspend event, and calls into the various
+ * kbase components to complete the suspend.
+ *
+ * Despite kbase_pm_suspend(), it will ignore to update Arbiter
+ * status if MALI_ARBITER_SUPPORT is enabled.
+ *
+ * @note the mechanisms used here rely on all user-space threads being frozen
+ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up
+ * the GPU e.g. via atom submission.
+ */
+void kbase_pm_driver_suspend(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_driver_resume() - Put GPU and driver in resume
+ * @param kbdev The kbase device structure for the device
+ * (must be a valid pointer)
+ *
+ * Resume the GPU, allow register accesses to it, and resume running atoms on
+ * the GPU.
+ *
+ * This is called in response to an OS resume event, and calls into the various
+ * kbase components to complete the resume.
+ *
+ * Also called when using VM arbiter, when GPU access has been granted.
+ *
+ * Despite kbase_pm_resume(), it will ignore to update Arbiter
+ * status if MALI_ARBITER_SUPPORT is enabled.
+ */
+void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start);
+
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
+/**
+ * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM
+ * @kbdev: Device pointer
+ *
+ * Handles the case that the Arbiter has forced the GPU away from the VM,
+ * so that interrupts will not be received and registers are no longer
+ * accessible because replaced by dummy RAM.
+ * Kill any running tasks and put the driver into a GPU powered-off state.
+ */
+void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev);
+#endif /* CONFIG_MALI_ARBITER_SUPPORT */
+
+#endif /* _KBASE_PM_H_ */
diff --git a/mali_kbase/mali_kbase_softjobs.c b/mali_kbase/mali_kbase_softjobs.c
index 45ce8ad..cbb0c76 100644
--- a/mali_kbase/mali_kbase_softjobs.c
+++ b/mali_kbase/mali_kbase_softjobs.c
@@ -34,6 +34,7 @@
#include <mali_kbase_hwaccess_time.h>
#include <mali_kbase_mem_linux.h>
#include <tl/mali_kbase_tracepoints.h>
+#include <mali_linux_trace.h>
#include <linux/version.h>
#include <linux/ktime.h>
#include <linux/pfn.h>
@@ -133,7 +134,7 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom)
{
struct kbase_vmap_struct map;
void *user_result;
- struct timespec ts;
+ struct timespec64 ts;
struct base_dump_cpu_gpu_counters data;
u64 system_time;
u64 cycle_counter;
@@ -916,6 +917,12 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx,
return 0;
}
+
+#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE)
+#define offsetofend(TYPE, MEMBER) \
+ (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER))
+#endif
+
/*
* Sizes of user data to copy for each just-in-time memory interface version
*
@@ -1000,7 +1007,7 @@ static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom)
katom->jit_blocked = false;
lockdep_assert_held(&kctx->jctx.lock);
- list_add_tail(&katom->jit_node, &kctx->jit_atoms_head);
+ list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head);
/*
* Note:
@@ -1037,7 +1044,7 @@ static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom)
struct list_head *target_list_head = NULL;
struct kbase_jd_atom *entry;
- list_for_each_entry(entry, &kctx->jit_pending_alloc, queue) {
+ list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) {
if (katom->age < entry->age) {
target_list_head = &entry->queue;
break;
@@ -1045,7 +1052,7 @@ static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom)
}
if (target_list_head == NULL)
- target_list_head = &kctx->jit_pending_alloc;
+ target_list_head = &kctx->jctx.jit_pending_alloc;
list_add_tail(&katom->queue, target_list_head);
}
@@ -1060,6 +1067,10 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom)
u64 *ptr, new_addr;
u32 count = katom->nr_extres;
u32 i;
+ bool ignore_pressure_limit = false;
+
+ trace_sysgraph(SGR_SUBMIT, kctx->id,
+ kbase_jd_atom_id(kctx, katom));
if (katom->jit_blocked) {
list_del(&katom->queue);
@@ -1080,6 +1091,15 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom)
}
}
+#if MALI_JIT_PRESSURE_LIMIT
+ /**
+ * If this is the only JIT_ALLOC atom in-flight then allow it to exceed
+ * the defined pressure limit.
+ */
+ if (kctx->jit_current_allocations == 0)
+ ignore_pressure_limit = true;
+#endif /* MALI_JIT_PRESSURE_LIMIT */
+
for (i = 0, info = katom->softjob_data; i < count; i++, info++) {
if (kctx->jit_alloc[info->id]) {
/* The JIT ID is duplicated in this atom. Roll back
@@ -1099,17 +1119,14 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom)
}
/* Create a JIT allocation */
- reg = kbase_jit_allocate(kctx, info);
+ reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit);
if (!reg) {
struct kbase_jd_atom *jit_atom;
bool can_block = false;
lockdep_assert_held(&kctx->jctx.lock);
- jit_atom = list_first_entry(&kctx->jit_atoms_head,
- struct kbase_jd_atom, jit_node);
-
- list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) {
+ list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) {
if (jit_atom == katom)
break;
@@ -1280,7 +1297,7 @@ static int kbase_jit_free_prepare(struct kbase_jd_atom *katom)
for (i = 0; i < count; i++)
KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]);
- list_add_tail(&katom->jit_node, &kctx->jit_atoms_head);
+ list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head);
return 0;
@@ -1335,7 +1352,7 @@ void kbase_jit_retry_pending_alloc(struct kbase_context *kctx)
LIST_HEAD(jit_pending_alloc_list);
struct list_head *i, *tmp;
- list_splice_tail_init(&kctx->jit_pending_alloc,
+ list_splice_tail_init(&kctx->jctx.jit_pending_alloc,
&jit_pending_alloc_list);
list_for_each_safe(i, tmp, &jit_pending_alloc_list) {
@@ -1363,7 +1380,7 @@ static void kbase_jit_free_finish(struct kbase_jd_atom *katom)
return;
}
- /* Remove this atom from the kctx->jit_atoms_head list */
+ /* Remove this atom from the jit_atoms_head list */
list_del(&katom->jit_node);
for (j = 0; j != katom->nr_extres; ++j) {
@@ -1522,6 +1539,9 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom)
KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom);
+ trace_sysgraph(SGR_SUBMIT, kctx->id,
+ kbase_jd_atom_id(kctx, katom));
+
switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
ret = kbase_dump_cpu_gpu_time(katom);
@@ -1684,6 +1704,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
void kbase_finish_soft_job(struct kbase_jd_atom *katom)
{
+ trace_sysgraph(SGR_COMPLETE, katom->kctx->id,
+ kbase_jd_atom_id(katom->kctx, katom));
+
switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
/* Nothing to do */
diff --git a/mali_kbase/mali_kbase_vinstr.c b/mali_kbase/mali_kbase_vinstr.c
index f01291a..d96b565 100644
--- a/mali_kbase/mali_kbase_vinstr.c
+++ b/mali_kbase/mali_kbase_vinstr.c
@@ -139,10 +139,7 @@ static const struct file_operations vinstr_client_fops = {
*/
static u64 kbasep_vinstr_timestamp_ns(void)
{
- struct timespec ts;
-
- getrawmonotonic(&ts);
- return (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
+ return ktime_get_raw_ns();
}
/**
diff --git a/mali_kbase/mali_linux_kbase_trace.h b/mali_kbase/mali_linux_kbase_trace.h
deleted file mode 100644
index 6c6a8c6..0000000
--- a/mali_kbase/mali_linux_kbase_trace.h
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2014,2018 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, you can access it online at
- * http://www.gnu.org/licenses/gpl-2.0.html.
- *
- * SPDX-License-Identifier: GPL-2.0
- *
- */
-
-
-#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _TRACE_MALI_KBASE_H
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM mali
-
-#include <linux/tracepoint.h>
-
-DECLARE_EVENT_CLASS(mali_slot_template,
- TP_PROTO(int jobslot, unsigned int info_val),
- TP_ARGS(jobslot, info_val),
- TP_STRUCT__entry(
- __field(unsigned int, jobslot)
- __field(unsigned int, info_val)
- ),
- TP_fast_assign(
- __entry->jobslot = jobslot;
- __entry->info_val = info_val;
- ),
- TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val)
-);
-
-#define DEFINE_MALI_SLOT_EVENT(name) \
-DEFINE_EVENT(mali_slot_template, mali_##name, \
- TP_PROTO(int jobslot, unsigned int info_val), \
- TP_ARGS(jobslot, info_val))
-DEFINE_MALI_SLOT_EVENT(JM_SUBMIT);
-DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE);
-DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD);
-DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD);
-DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP);
-DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0);
-DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1);
-DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP);
-DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0);
-DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1);
-DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP);
-DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT);
-DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER);
-DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER);
-DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED);
-DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED);
-DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT);
-DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB);
-DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED);
-DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED);
-DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED);
-DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE);
-DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB);
-DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED);
-DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB);
-DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ);
-#undef DEFINE_MALI_SLOT_EVENT
-
-DECLARE_EVENT_CLASS(mali_refcount_template,
- TP_PROTO(int refcount, unsigned int info_val),
- TP_ARGS(refcount, info_val),
- TP_STRUCT__entry(
- __field(unsigned int, refcount)
- __field(unsigned int, info_val)
- ),
- TP_fast_assign(
- __entry->refcount = refcount;
- __entry->info_val = info_val;
- ),
- TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val)
-);
-
-#define DEFINE_MALI_REFCOUNT_EVENT(name) \
-DEFINE_EVENT(mali_refcount_template, mali_##name, \
- TP_PROTO(int refcount, unsigned int info_val), \
- TP_ARGS(refcount, info_val))
-DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK);
-DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB);
-DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB);
-DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX);
-DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS);
-DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE);
-DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE);
-#undef DEFINE_MALI_REFCOUNT_EVENT
-
-DECLARE_EVENT_CLASS(mali_add_template,
- TP_PROTO(int gpu_addr, unsigned int info_val),
- TP_ARGS(gpu_addr, info_val),
- TP_STRUCT__entry(
- __field(unsigned int, gpu_addr)
- __field(unsigned int, info_val)
- ),
- TP_fast_assign(
- __entry->gpu_addr = gpu_addr;
- __entry->info_val = info_val;
- ),
- TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val)
-);
-
-#define DEFINE_MALI_ADD_EVENT(name) \
-DEFINE_EVENT(mali_add_template, mali_##name, \
- TP_PROTO(int gpu_addr, unsigned int info_val), \
- TP_ARGS(gpu_addr, info_val))
-DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY);
-DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR);
-DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES);
-DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER);
-DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END);
-DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER);
-DEFINE_MALI_ADD_EVENT(JD_DONE);
-DEFINE_MALI_ADD_EVENT(JD_CANCEL);
-DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT);
-DEFINE_MALI_ADD_EVENT(JM_IRQ);
-DEFINE_MALI_ADD_EVENT(JM_IRQ_END);
-DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS);
-DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE);
-DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED);
-DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED);
-DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE);
-DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET);
-DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE);
-DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL);
-DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL);
-DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX);
-DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX);
-DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END);
-DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START);
-DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB);
-DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED);
-DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP);
-DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP);
-DEFINE_MALI_ADD_EVENT(PM_PWRON);
-DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER);
-DEFINE_MALI_ADD_EVENT(PM_PWRON_L2);
-DEFINE_MALI_ADD_EVENT(PM_PWROFF);
-DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER);
-DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2);
-DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED);
-DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER);
-DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2);
-DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED);
-DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER);
-DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED);
-DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED);
-DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED);
-DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED);
-DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE);
-DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER);
-DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE);
-DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER);
-DEFINE_MALI_ADD_EVENT(PM_GPU_ON);
-DEFINE_MALI_ADD_EVENT(PM_GPU_OFF);
-DEFINE_MALI_ADD_EVENT(PM_SET_POLICY);
-DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT);
-DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM);
-DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY);
-DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS);
-#undef DEFINE_MALI_ADD_EVENT
-
-#endif /* _TRACE_MALI_KBASE_H */
-
-#undef TRACE_INCLUDE_PATH
-#undef linux
-#define TRACE_INCLUDE_PATH .
-#undef TRACE_INCLUDE_FILE
-#define TRACE_INCLUDE_FILE mali_linux_kbase_trace
-
-/* This part must be outside protection */
-#include <trace/define_trace.h>
diff --git a/mali_kbase/mali_linux_trace.h b/mali_kbase/mali_linux_trace.h
index b639764..f618755 100644
--- a/mali_kbase/mali_linux_trace.h
+++ b/mali_kbase/mali_linux_trace.h
@@ -288,7 +288,7 @@ DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free,
TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx",
__entry->start_addr, __entry->nr_pages, __entry->backed_pages));
-#if MALI_JIT_PRESSURE_LIMIT && !MALI_USE_CSF
+#if MALI_JIT_PRESSURE_LIMIT
/* trace_mali_jit_report
*
* Tracepoint about the GPU data structure read to form a just-in-time memory
@@ -326,9 +326,11 @@ TRACE_EVENT(mali_jit_report,
),
__entry->read_val, __entry->used_pages)
);
-#endif /* MALI_JIT_PRESSURE_LIMIT && !MALI_USE_CSF */
+#endif /* MALI_JIT_PRESSURE_LIMIT */
+#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE)
TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE);
+#endif
#if MALI_JIT_PRESSURE_LIMIT
/* trace_mali_jit_report_pressure
@@ -366,6 +368,70 @@ TRACE_EVENT(mali_jit_report_pressure,
);
#endif /* MALI_JIT_PRESSURE_LIMIT */
+#ifndef __TRACE_SYSGRAPH_ENUM
+#define __TRACE_SYSGRAPH_ENUM
+/* Enum of sysgraph message IDs */
+enum sysgraph_msg {
+ SGR_ARRIVE,
+ SGR_DEP_RES,
+ SGR_SUBMIT,
+ SGR_COMPLETE,
+ SGR_POST,
+ SGR_ACTIVE,
+ SGR_INACTIVE
+};
+#endif /* __TRACE_SYSGRAPH_ENUM */
+
+/* A template for SYSGRAPH events
+ *
+ * Most of the sysgraph events contain only one input argument
+ * which is atom_id therefore they will be using a common template
+ */
+TRACE_EVENT(sysgraph,
+ TP_PROTO(enum sysgraph_msg message, unsigned int proc_id,
+ unsigned int atom_id),
+ TP_ARGS(message, proc_id, atom_id),
+ TP_STRUCT__entry(
+ __field(unsigned int, proc_id)
+ __field(enum sysgraph_msg, message)
+ __field(unsigned int, atom_id)
+ ),
+ TP_fast_assign(
+ __entry->proc_id = proc_id;
+ __entry->message = message;
+ __entry->atom_id = atom_id;
+ ),
+ TP_printk("msg=%u proc_id=%u, param1=%d\n", __entry->message,
+ __entry->proc_id, __entry->atom_id)
+);
+
+/* A template for SYSGRAPH GPU events
+ *
+ * Sysgraph events that record start/complete events
+ * on GPU also record a js value in addition to the
+ * atom id.
+ */
+TRACE_EVENT(sysgraph_gpu,
+ TP_PROTO(enum sysgraph_msg message, unsigned int proc_id,
+ unsigned int atom_id, unsigned int js),
+ TP_ARGS(message, proc_id, atom_id, js),
+ TP_STRUCT__entry(
+ __field(unsigned int, proc_id)
+ __field(enum sysgraph_msg, message)
+ __field(unsigned int, atom_id)
+ __field(unsigned int, js)
+ ),
+ TP_fast_assign(
+ __entry->proc_id = proc_id;
+ __entry->message = message;
+ __entry->atom_id = atom_id;
+ __entry->js = js;
+ ),
+ TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d\n",
+ __entry->message, __entry->proc_id,
+ __entry->atom_id, __entry->js)
+);
+
/* Tracepoint files get included more than once - protect against multiple
* definition
*/
@@ -450,9 +516,14 @@ TRACE_EVENT(mali_jit_trim,
TP_printk("freed_pages=%zu", __entry->freed_pages)
);
+#include "mali_kbase_debug_linux_ktrace.h"
+
#endif /* _TRACE_MALI_H */
#undef TRACE_INCLUDE_PATH
+/* lwn.net/Articles/383362 suggests this should remain as '.', and instead
+ * extend CFLAGS
+ */
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE mali_linux_trace
diff --git a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c
index 46800fe..2d8fb51 100644
--- a/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c
+++ b/mali_kbase/mmu/backend/mali_kbase_mmu_jm.c
@@ -30,6 +30,7 @@
#include <backend/gpu/mali_kbase_device_internal.h>
#include <mali_kbase_as_fault_debugfs.h>
#include "../mali_kbase_mmu_internal.h"
+#include "mali_kbase_device_internal.h"
void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut,
struct kbase_mmu_setup * const setup)
@@ -295,6 +296,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
unsigned long flags;
u32 new_mask;
u32 tmp, bf_bits, pf_bits;
+ bool gpu_lost = false;
dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n",
__func__, irq_stat);
@@ -338,7 +340,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
* and a job causing the Bus/Page fault shouldn't complete until
* the MMU is updated
*/
- kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no);
+ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no);
/* find faulting address */
fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no,
@@ -369,6 +371,14 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
MMU_AS_REG(as_no, AS_FAULTEXTRA_LO));
}
+ /* check if we still have GPU */
+ gpu_lost = kbase_is_gpu_lost(kbdev);
+ if (gpu_lost) {
+ if (kctx)
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
+ return;
+ }
+
if (kbase_as_has_bus_fault(as, fault)) {
/* Mark bus fault as handled.
* Note that a bus fault is processed first in case
diff --git a/mali_kbase/mmu/mali_kbase_mmu.c b/mali_kbase/mmu/mali_kbase_mmu.c
index c4bea39..db27832 100644
--- a/mali_kbase/mmu/mali_kbase_mmu.c
+++ b/mali_kbase/mmu/mali_kbase_mmu.c
@@ -32,6 +32,7 @@
#include <gpu/mali_kbase_gpu_regmap.h>
#include <tl/mali_kbase_tracepoints.h>
#include <mali_kbase_instr_defs.h>
+#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_debug.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_hw.h>
@@ -517,6 +518,13 @@ static bool page_fault_try_alloc(struct kbase_context *kctx,
return true;
}
+/* Small wrapper function to factor out GPU-dependent context releasing */
+static void release_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
+}
+
void page_fault_worker(struct work_struct *data)
{
u64 fault_pfn;
@@ -552,8 +560,8 @@ void page_fault_worker(struct work_struct *data)
* Therefore, it cannot be scheduled out of this AS until we explicitly
* release it
*/
- kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
- if (WARN_ON(!kctx)) {
+ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no);
+ if (!kctx) {
atomic_dec(&kbdev->faults_pending);
return;
}
@@ -917,7 +925,7 @@ fault_done:
* By this point, the fault was handled in some way,
* so release the ctx refcount
*/
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ release_ctx(kbdev, kctx);
atomic_dec(&kbdev->faults_pending);
dev_dbg(kbdev->dev, "Leaving page_fault_worker %p\n", (void *)data);
@@ -1562,7 +1570,7 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
kbdev = kctx->kbdev;
mutex_lock(&kbdev->js_data.queue_mutex);
- ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx);
+ ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx);
mutex_unlock(&kbdev->js_data.queue_mutex);
if (ctx_is_in_runpool) {
@@ -1571,7 +1579,7 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[kctx->as_nr],
vpfn, nr, sync);
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ release_ctx(kbdev, kctx);
}
}
@@ -2146,8 +2154,8 @@ void bus_fault_worker(struct work_struct *data)
* flagging of the bus-fault. Therefore, it cannot be scheduled out of
* this AS until we explicitly release it
*/
- kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
- if (WARN_ON(!kctx)) {
+ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no);
+ if (!kctx) {
atomic_dec(&kbdev->faults_pending);
return;
}
@@ -2157,7 +2165,7 @@ void bus_fault_worker(struct work_struct *data)
"Permission failure", fault);
kbase_mmu_hw_clear_fault(kbdev, faulting_as,
KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ release_ctx(kbdev, kctx);
atomic_dec(&kbdev->faults_pending);
return;
@@ -2172,7 +2180,7 @@ void bus_fault_worker(struct work_struct *data)
kbase_pm_context_idle(kbdev);
}
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ release_ctx(kbdev, kctx);
atomic_dec(&kbdev->faults_pending);
}
diff --git a/mali_kbase/tests/Mconfig b/mali_kbase/tests/Mconfig
index af4e383..be3fedb 100644
--- a/mali_kbase/tests/Mconfig
+++ b/mali_kbase/tests/Mconfig
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2018 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -30,3 +30,9 @@ config BUILD_CSF_TESTS
bool
default y if UNIT_TEST_KERNEL_MODULES && GPU_HAS_CSF
default n
+
+config BUILD_ARBIF_TESTS
+ bool
+ default y if UNIT_TEST_KERNEL_MODULES && MALI_ARBITER_SUPPORT
+ default n
+
diff --git a/mali_kbase/tests/mali_kutf_irq_test/Makefile b/mali_kbase/tests/mali_kutf_irq_test/Makefile
index 9218a40..bc4d654 100644
--- a/mali_kbase/tests/mali_kutf_irq_test/Makefile
+++ b/mali_kbase/tests/mali_kutf_irq_test/Makefile
@@ -1,5 +1,5 @@
#
-# (C) COPYRIGHT 2015, 2017-2018 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2015, 2017-2018, 2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@@ -38,6 +38,8 @@ TEST_CCFLAGS := \
-I$(CURDIR)/../../../ \
-I$(CURDIR)/../../ \
-I$(CURDIR)/../../backend/gpu \
+ -I$(CURDIR)/../../debug \
+ -I$(CURDIR)/../../debug/backend \
-I$(CURDIR)/ \
-I$(srctree)/drivers/staging/android \
-I$(srctree)/include/linux
diff --git a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c
index a220a46..26b442a 100644
--- a/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c
+++ b/mali_kbase/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c
@@ -58,10 +58,10 @@ struct kutf_irq_fixture_data {
#define NR_TEST_IRQS ((u32)1000000)
-/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not
- * expect to see this in normal use (e.g., when Android is running).
+/* IRQ for the test to trigger. Currently POWER_CHANGED_SINGLE as it is
+ * otherwise unused in the DDK
*/
-#define TEST_IRQ MULTIPLE_GPU_FAULTS
+#define TEST_IRQ POWER_CHANGED_SINGLE
#define IRQ_TIMEOUT HZ
@@ -92,11 +92,11 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data)
struct kbase_device *kbdev = kbase_untag(data);
u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS));
irqreturn_t result;
- struct timespec tval;
+ u64 tval;
bool has_test_irq = val & TEST_IRQ;
if (has_test_irq) {
- getnstimeofday(&tval);
+ tval = ktime_get_real_ns();
/* Clear the test source only here */
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
TEST_IRQ);
@@ -107,7 +107,7 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data)
result = kbase_gpu_irq_test_handler(irq, data, val);
if (has_test_irq) {
- irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec);
+ irq_time = tval;
triggered = true;
wake_up(&wait);
result = IRQ_HANDLED;
@@ -191,12 +191,9 @@ static void mali_kutf_irq_latency(struct kutf_context *context)
GPU_IRQ_HANDLER);
for (i = 1; i <= NR_TEST_IRQS; i++) {
- struct timespec tval;
- u64 start_time;
+ u64 start_time = ktime_get_real_ns();
triggered = false;
- getnstimeofday(&tval);
- start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec);
/* Trigger fake IRQ */
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
diff --git a/mali_kbase/tl/mali_kbase_timeline.c b/mali_kbase/tl/mali_kbase_timeline.c
index 5d073be..88fba83 100644
--- a/mali_kbase/tl/mali_kbase_timeline.c
+++ b/mali_kbase/tl/mali_kbase_timeline.c
@@ -105,12 +105,12 @@ static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer)
/*****************************************************************************/
int kbase_timeline_init(struct kbase_timeline **timeline,
- atomic_t *timeline_is_enabled)
+ atomic_t *timeline_flags)
{
enum tl_stream_type i;
struct kbase_timeline *result;
- if (!timeline || !timeline_is_enabled)
+ if (!timeline || !timeline_flags)
return -EINVAL;
result = kzalloc(sizeof(*result), GFP_KERNEL);
@@ -129,7 +129,7 @@ int kbase_timeline_init(struct kbase_timeline **timeline,
atomic_set(&result->autoflush_timer_active, 0);
kbase_timer_setup(&result->autoflush_timer,
kbasep_timeline_autoflush_timer_callback);
- result->is_enabled = timeline_is_enabled;
+ result->timeline_flags = timeline_flags;
*timeline = result;
@@ -176,10 +176,10 @@ static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev)
int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags)
{
int ret;
- u32 tlstream_enabled = TLSTREAM_ENABLED | flags;
+ u32 timeline_flags = TLSTREAM_ENABLED | flags;
struct kbase_timeline *timeline = kbdev->timeline;
- if (!atomic_cmpxchg(timeline->is_enabled, 0, tlstream_enabled)) {
+ if (!atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) {
int rcode;
ret = anon_inode_getfd(
@@ -188,7 +188,7 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags)
timeline,
O_RDONLY | O_CLOEXEC);
if (ret < 0) {
- atomic_set(timeline->is_enabled, 0);
+ atomic_set(timeline->timeline_flags, 0);
return ret;
}
diff --git a/mali_kbase/tl/mali_kbase_timeline.h b/mali_kbase/tl/mali_kbase_timeline.h
index d800288..cd48411 100644
--- a/mali_kbase/tl/mali_kbase_timeline.h
+++ b/mali_kbase/tl/mali_kbase_timeline.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -31,15 +31,15 @@ struct kbase_timeline;
/**
* kbase_timeline_init - initialize timeline infrastructure in kernel
- * @timeline: Newly created instance of kbase_timeline will
- * be stored in this pointer.
- * @timeline_is_enabled: Timeline status will be written to this variable
- * when a client is attached/detached. The variable
- * must be valid while timeline instance is valid.
+ * @timeline: Newly created instance of kbase_timeline will be stored in
+ * this pointer.
+ * @timeline_flags: Timeline status will be written to this variable when a
+ * client is attached/detached. The variable must be valid
+ * while timeline instance is valid.
* Return: zero on success, negative number on error
*/
int kbase_timeline_init(struct kbase_timeline **timeline,
- atomic_t *timeline_is_enabled);
+ atomic_t *timeline_flags);
/**
* kbase_timeline_term - terminate timeline infrastructure in kernel
diff --git a/mali_kbase/tl/mali_kbase_timeline_io.c b/mali_kbase/tl/mali_kbase_timeline_io.c
index 6e09a17..cdde928 100644
--- a/mali_kbase/tl/mali_kbase_timeline_io.c
+++ b/mali_kbase/tl/mali_kbase_timeline_io.c
@@ -324,6 +324,6 @@ static int kbasep_timeline_io_release(struct inode *inode, struct file *filp)
atomic_set(&timeline->autoflush_timer_active, 0);
del_timer_sync(&timeline->autoflush_timer);
- atomic_set(timeline->is_enabled, 0);
+ atomic_set(timeline->timeline_flags, 0);
return 0;
}
diff --git a/mali_kbase/tl/mali_kbase_timeline_priv.h b/mali_kbase/tl/mali_kbase_timeline_priv.h
index 73499ce..3596584 100644
--- a/mali_kbase/tl/mali_kbase_timeline_priv.h
+++ b/mali_kbase/tl/mali_kbase_timeline_priv.h
@@ -40,7 +40,7 @@
* have access to the timeline streams at any given time.
* @event_queue: Timeline stream event queue
* @bytes_collected: Number of bytes read by user
- * @is_enabled: Zero, if timeline is disabled. Timeline stream flags
+ * @timeline_flags: Zero, if timeline is disabled. Timeline stream flags
* otherwise. See kbase_timeline_io_acquire().
* @obj_header_btc: Remaining bytes to copy for the object stream header
* @aux_header_btc: Remaining bytes to copy for the aux stream header
@@ -54,7 +54,7 @@ struct kbase_timeline {
#if MALI_UNIT_TEST
atomic_t bytes_collected;
#endif /* MALI_UNIT_TEST */
- atomic_t *is_enabled;
+ atomic_t *timeline_flags;
size_t obj_header_btc;
size_t aux_header_btc;
};
diff --git a/mali_kbase/tl/mali_kbase_tl_serialize.h b/mali_kbase/tl/mali_kbase_tl_serialize.h
index 90808ce..3e37827 100644
--- a/mali_kbase/tl/mali_kbase_tl_serialize.h
+++ b/mali_kbase/tl/mali_kbase_tl_serialize.h
@@ -1,6 +1,6 @@
/*
*
- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -113,11 +113,9 @@ static inline size_t kbasep_serialize_string(
*/
static inline size_t kbasep_serialize_timestamp(void *buffer, size_t pos)
{
- struct timespec ts;
u64 timestamp;
- getrawmonotonic(&ts);
- timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
+ timestamp = ktime_get_raw_ns();
return kbasep_serialize_bytes(
buffer, pos,
diff --git a/mali_kbase/tl/mali_kbase_trace_defs.h b/mali_kbase/tl/mali_kbase_trace_defs.h
deleted file mode 100644
index 1ee6a59..0000000
--- a/mali_kbase/tl/mali_kbase_trace_defs.h
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- *
- * (C) COPYRIGHT 2011-2015,2018-2019 ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, you can access it online at
- * http://www.gnu.org/licenses/gpl-2.0.html.
- *
- * SPDX-License-Identifier: GPL-2.0
- *
- */
-
-
-
-/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE *****
- * ***** DO NOT INCLUDE DIRECTLY *****
- * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */
-
-/*
- * The purpose of this header file is just to contain a list of trace code idenitifers
- *
- * Each identifier is wrapped in a macro, so that its string form and enum form can be created
- *
- * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block.
- *
- * This allows automatic creation of an enum and a corresponding array of strings
- *
- * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE.
- * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE.
- *
- * e.g.:
- * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X
- * typedef enum
- * {
- * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X )
- * #include "mali_kbase_trace_defs.h"
- * #undef KBASE_TRACE_CODE_MAKE_CODE
- * } kbase_trace_code;
- *
- * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE
- *
- *
- * The use of the macro here is:
- * - KBASE_TRACE_CODE_MAKE_CODE( X )
- *
- * Which produces:
- * - For an enum, KBASE_TRACE_CODE_X
- * - For a string, "X"
- *
- *
- * For example:
- * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to:
- * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum
- * - "JM_JOB_COMPLETE" for the string
- * - To use it to trace an event, do:
- * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val );
- */
-
-#if 0 /* Dummy section to avoid breaking formatting */
-int dummy_array[] = {
-#endif
-
-/*
- * Core events
- */
- /* no info_val, no gpu_addr, no atom */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY),
- /* no info_val, no gpu_addr, no atom */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM),
- /* info_val == GPU_IRQ_STATUS register */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ),
- /* info_val == bits cleared */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR),
- /* info_val == GPU_IRQ_STATUS register */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE),
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET),
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET),
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR),
- /* GPU addr==dump address */
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE),
- KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES),
-/*
- * Job Slot management events
- */
- /* info_val==irq rawstat at start */
- KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ),
- /* info_val==jobs processed */
- KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END),
-/* In the following:
- *
- * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases)
- * - uatom==kernel-side mapped uatom address (for correlation with user-side)
- */
- /* info_val==exit code; gpu_addr==chain gpuaddr */
- KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE),
- /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT),
- /* gpu_addr is as follows:
- * - If JS_STATUS active after soft-stop, val==gpu addr written to
- * JS_HEAD on submit
- * - otherwise gpu_addr==0 */
- KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP),
- KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0),
- KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1),
- /* gpu_addr==JS_HEAD read */
- KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP),
- /* gpu_addr==JS_HEAD read */
- KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0),
- /* gpu_addr==JS_HEAD read */
- KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1),
- /* gpu_addr==JS_TAIL read */
- KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD),
-/* gpu_addr is as follows:
- * - If JS_STATUS active before soft-stop, val==JS_HEAD
- * - otherwise gpu_addr==0
- */
- /* gpu_addr==JS_HEAD read */
- KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD),
- KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS),
- KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE),
- /* info_val == is_scheduled */
- KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED),
- /* info_val == is_scheduled */
- KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED),
- KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE),
- /* info_val == nr jobs submitted */
- KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP),
- /* gpu_addr==JS_HEAD_NEXT last written */
- KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT),
- KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET),
- KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER),
- KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER),
-/*
- * Job dispatch events
- */
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_DONE),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB),
- /* gpu_addr==0, info_val==0, uatom==0 */
- KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER),
-/*
- * Scheduler Core events
- */
- KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB),
- /* gpu_addr==last value written/would be written to JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB),
- KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB),
- /* gpu_addr==value to write into JS_HEAD */
- KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED),
- KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED),
- /* info_val == lower 32 bits of affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT),
- /* info_val == lower 32 bits of affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED),
- /* info_val == lower 32 bits of affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED),
- /* info_val == lower 32 bits of rechecked affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED),
- /* info_val == lower 32 bits of rechecked affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED),
- /* info_val == lower 32 bits of affinity */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE),
- /* info_val == the ctx attribute now on ctx */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX),
- /* info_val == the ctx attribute now on runpool */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL),
- /* info_val == the ctx attribute now off ctx */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX),
- /* info_val == the ctx attribute now off runpool */
- KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL),
-/*
- * Scheduler Policy events
- */
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX),
- /* info_val == whether it was evicted */
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ),
- /* gpu_addr==JS_HEAD to write if the job were run */
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START),
- KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END),
-/*
- * Power Management Events
- */
- KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP),
- KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER),
- /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */
- KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER),
- KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED),
- KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE),
- KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON),
- KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF),
- /* info_val == policy number, or -1 for "Already changing" */
- KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY),
- KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY),
- /* info_val == policy number */
- KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT),
- /* info_val == policy number */
- KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM),
-/* Unused code just to make it easier to not have a comma at the end.
- * All other codes MUST come before this */
- KBASE_TRACE_CODE_MAKE_CODE(DUMMY)
-
-#if 0 /* Dummy section to avoid breaking formatting */
-};
-#endif
-
-/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */
diff --git a/mali_kbase/tl/mali_kbase_tracepoints.c b/mali_kbase/tl/mali_kbase_tracepoints.c
index b028ef8..e445a3a 100644
--- a/mali_kbase/tl/mali_kbase_tracepoints.c
+++ b/mali_kbase/tl/mali_kbase_tracepoints.c
@@ -66,12 +66,18 @@ enum tl_msg_id_obj {
KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE,
KBASE_TL_EVENT_ATOM_SOFTJOB_START,
KBASE_TL_EVENT_ATOM_SOFTJOB_END,
+ KBASE_TL_EVENT_ARB_GRANTED,
+ KBASE_TL_EVENT_ARB_STARTED,
+ KBASE_TL_EVENT_ARB_STOP_REQUESTED,
+ KBASE_TL_EVENT_ARB_STOPPED,
KBASE_JD_GPU_SOFT_RESET,
KBASE_TL_KBASE_NEW_DEVICE,
KBASE_TL_KBASE_DEVICE_PROGRAM_CSG,
KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG,
KBASE_TL_KBASE_NEW_CTX,
KBASE_TL_KBASE_DEL_CTX,
+ KBASE_TL_KBASE_CTX_ASSIGN_AS,
+ KBASE_TL_KBASE_CTX_UNASSIGN_AS,
KBASE_TL_KBASE_NEW_KCPUQUEUE,
KBASE_TL_KBASE_DEL_KCPUQUEUE,
KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL,
@@ -270,14 +276,30 @@ enum tl_msg_id_aux {
"atom soft job has completed", \
"@p", \
"atom") \
+ TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_GRANTED, \
+ "Arbiter has granted gpu access", \
+ "@p", \
+ "gpu") \
+ TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STARTED, \
+ "Driver is running again and able to process jobs", \
+ "@p", \
+ "gpu") \
+ TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOP_REQUESTED, \
+ "Arbiter has requested driver to stop using gpu", \
+ "@p", \
+ "gpu") \
+ TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOPPED, \
+ "Driver has stopped using gpu", \
+ "@p", \
+ "gpu") \
TRACEPOINT_DESC(KBASE_JD_GPU_SOFT_RESET, \
"gpu soft reset", \
"@p", \
"gpu") \
TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \
"New KBase Device", \
- "@III", \
- "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs") \
+ "@IIII", \
+ "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs,kbase_device_as_count") \
TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, \
"CSG is programmed to a slot", \
"@III", \
@@ -294,6 +316,14 @@ enum tl_msg_id_aux {
"Delete KBase Context", \
"@I", \
"kernel_ctx_id") \
+ TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_ASSIGN_AS, \
+ "Address Space is assigned to a KBase context", \
+ "@II", \
+ "kernel_ctx_id,kbase_device_as_index") \
+ TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_UNASSIGN_AS, \
+ "Address Space is unassigned from a KBase context", \
+ "@I", \
+ "kernel_ctx_id") \
TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_KCPUQUEUE, \
"New KCPU Queue", \
"@pII", \
@@ -1463,6 +1493,94 @@ void __kbase_tlstream_tl_event_atom_softjob_end(
kbase_tlstream_msgbuf_release(stream, acq_flags);
}
+void __kbase_tlstream_tl_event_arb_granted(
+ struct kbase_tlstream *stream,
+ const void *gpu)
+{
+ const u32 msg_id = KBASE_TL_EVENT_ARB_GRANTED;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(gpu)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &gpu, sizeof(gpu));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
+void __kbase_tlstream_tl_event_arb_started(
+ struct kbase_tlstream *stream,
+ const void *gpu)
+{
+ const u32 msg_id = KBASE_TL_EVENT_ARB_STARTED;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(gpu)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &gpu, sizeof(gpu));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
+void __kbase_tlstream_tl_event_arb_stop_requested(
+ struct kbase_tlstream *stream,
+ const void *gpu)
+{
+ const u32 msg_id = KBASE_TL_EVENT_ARB_STOP_REQUESTED;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(gpu)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &gpu, sizeof(gpu));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
+void __kbase_tlstream_tl_event_arb_stopped(
+ struct kbase_tlstream *stream,
+ const void *gpu)
+{
+ const u32 msg_id = KBASE_TL_EVENT_ARB_STOPPED;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(gpu)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &gpu, sizeof(gpu));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
void __kbase_tlstream_jd_gpu_soft_reset(
struct kbase_tlstream *stream,
const void *gpu)
@@ -1757,13 +1875,15 @@ void __kbase_tlstream_tl_kbase_new_device(
struct kbase_tlstream *stream,
u32 kbase_device_id,
u32 kbase_device_gpu_core_count,
- u32 kbase_device_max_num_csgs)
+ u32 kbase_device_max_num_csgs,
+ u32 kbase_device_as_count)
{
const u32 msg_id = KBASE_TL_KBASE_NEW_DEVICE;
const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ sizeof(kbase_device_id)
+ sizeof(kbase_device_gpu_core_count)
+ sizeof(kbase_device_max_num_csgs)
+ + sizeof(kbase_device_as_count)
;
char *buffer;
unsigned long acq_flags;
@@ -1779,6 +1899,8 @@ void __kbase_tlstream_tl_kbase_new_device(
pos, &kbase_device_gpu_core_count, sizeof(kbase_device_gpu_core_count));
pos = kbasep_serialize_bytes(buffer,
pos, &kbase_device_max_num_csgs, sizeof(kbase_device_max_num_csgs));
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &kbase_device_as_count, sizeof(kbase_device_as_count));
kbase_tlstream_msgbuf_release(stream, acq_flags);
}
@@ -1887,6 +2009,54 @@ void __kbase_tlstream_tl_kbase_del_ctx(
kbase_tlstream_msgbuf_release(stream, acq_flags);
}
+void __kbase_tlstream_tl_kbase_ctx_assign_as(
+ struct kbase_tlstream *stream,
+ u32 kernel_ctx_id,
+ u32 kbase_device_as_index)
+{
+ const u32 msg_id = KBASE_TL_KBASE_CTX_ASSIGN_AS;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(kernel_ctx_id)
+ + sizeof(kbase_device_as_index)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &kernel_ctx_id, sizeof(kernel_ctx_id));
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &kbase_device_as_index, sizeof(kbase_device_as_index));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
+void __kbase_tlstream_tl_kbase_ctx_unassign_as(
+ struct kbase_tlstream *stream,
+ u32 kernel_ctx_id)
+{
+ const u32 msg_id = KBASE_TL_KBASE_CTX_UNASSIGN_AS;
+ const size_t msg_size = sizeof(msg_id) + sizeof(u64)
+ + sizeof(kernel_ctx_id)
+ ;
+ char *buffer;
+ unsigned long acq_flags;
+ size_t pos = 0;
+
+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags);
+
+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_serialize_timestamp(buffer, pos);
+ pos = kbasep_serialize_bytes(buffer,
+ pos, &kernel_ctx_id, sizeof(kernel_ctx_id));
+
+ kbase_tlstream_msgbuf_release(stream, acq_flags);
+}
+
void __kbase_tlstream_tl_kbase_new_kcpuqueue(
struct kbase_tlstream *stream,
const void *kcpu_queue,
diff --git a/mali_kbase/tl/mali_kbase_tracepoints.h b/mali_kbase/tl/mali_kbase_tracepoints.h
index fa2c399..7ea8ba8 100644
--- a/mali_kbase/tl/mali_kbase_tracepoints.h
+++ b/mali_kbase/tl/mali_kbase_tracepoints.h
@@ -56,8 +56,8 @@ extern const size_t __aux_stream_offset;
* relies on offset variables:
* __obj_stream_offset and __aux_stream_offset.
*/
-#define __TL_DISPATCH_STREAM(kbdev, stype) \
- ((struct kbase_tlstream *) \
+#define __TL_DISPATCH_STREAM(kbdev, stype) \
+ ((struct kbase_tlstream *) \
((u8 *)kbdev->timeline + __ ## stype ## _stream_offset))
struct tp_desc;
@@ -226,6 +226,18 @@ void __kbase_tlstream_tl_event_atom_softjob_start(
void __kbase_tlstream_tl_event_atom_softjob_end(
struct kbase_tlstream *stream,
const void *atom);
+void __kbase_tlstream_tl_event_arb_granted(
+ struct kbase_tlstream *stream,
+ const void *gpu);
+void __kbase_tlstream_tl_event_arb_started(
+ struct kbase_tlstream *stream,
+ const void *gpu);
+void __kbase_tlstream_tl_event_arb_stop_requested(
+ struct kbase_tlstream *stream,
+ const void *gpu);
+void __kbase_tlstream_tl_event_arb_stopped(
+ struct kbase_tlstream *stream,
+ const void *gpu);
void __kbase_tlstream_jd_gpu_soft_reset(
struct kbase_tlstream *stream,
const void *gpu);
@@ -275,7 +287,8 @@ void __kbase_tlstream_tl_kbase_new_device(
struct kbase_tlstream *stream,
u32 kbase_device_id,
u32 kbase_device_gpu_core_count,
- u32 kbase_device_max_num_csgs);
+ u32 kbase_device_max_num_csgs,
+ u32 kbase_device_as_count);
void __kbase_tlstream_tl_kbase_device_program_csg(
struct kbase_tlstream *stream,
u32 kbase_device_id,
@@ -292,6 +305,13 @@ void __kbase_tlstream_tl_kbase_new_ctx(
void __kbase_tlstream_tl_kbase_del_ctx(
struct kbase_tlstream *stream,
u32 kernel_ctx_id);
+void __kbase_tlstream_tl_kbase_ctx_assign_as(
+ struct kbase_tlstream *stream,
+ u32 kernel_ctx_id,
+ u32 kbase_device_as_index);
+void __kbase_tlstream_tl_kbase_ctx_unassign_as(
+ struct kbase_tlstream *stream,
+ u32 kernel_ctx_id);
void __kbase_tlstream_tl_kbase_new_kcpuqueue(
struct kbase_tlstream *stream,
const void *kcpu_queue,
@@ -465,10 +485,10 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NEW_CTX -
* object ctx is created
*
- * @kbdev: Kbase device
- * @ctx: Name of the context object
- * @ctx_nr: Kernel context number
- * @tgid: Thread Group Id
+ * @kbdev: Kbase device
+ * @ctx: Name of the context object
+ * @ctx_nr: Kernel context number
+ * @tgid: Thread Group Id
*/
#define KBASE_TLSTREAM_TL_NEW_CTX( \
kbdev, \
@@ -477,10 +497,10 @@ struct kbase_tlstream;
tgid \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_new_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
ctx, ctx_nr, tgid); \
} while (0)
@@ -488,10 +508,10 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NEW_GPU -
* object gpu is created
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
- * @gpu_id: Name of the GPU object
- * @core_count: Number of cores this GPU hosts
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
+ * @gpu_id: Name of the GPU object
+ * @core_count: Number of cores this GPU hosts
*/
#define KBASE_TLSTREAM_TL_NEW_GPU( \
kbdev, \
@@ -500,10 +520,10 @@ struct kbase_tlstream;
core_count \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_new_gpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
gpu, gpu_id, core_count); \
} while (0)
@@ -511,10 +531,10 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NEW_LPU -
* object lpu is created
*
- * @kbdev: Kbase device
- * @lpu: Name of the Logical Processing Unit object
- * @lpu_nr: Sequential number assigned to the newly created LPU
- * @lpu_fn: Property describing functional abilities of this LPU
+ * @kbdev: Kbase device
+ * @lpu: Name of the Logical Processing Unit object
+ * @lpu_nr: Sequential number assigned to the newly created LPU
+ * @lpu_fn: Property describing functional abilities of this LPU
*/
#define KBASE_TLSTREAM_TL_NEW_LPU( \
kbdev, \
@@ -523,10 +543,10 @@ struct kbase_tlstream;
lpu_fn \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_new_lpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
lpu, lpu_nr, lpu_fn); \
} while (0)
@@ -534,9 +554,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NEW_ATOM -
* object atom is created
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @atom_nr: Sequential number of an atom
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @atom_nr: Sequential number of an atom
*/
#define KBASE_TLSTREAM_TL_NEW_ATOM( \
kbdev, \
@@ -544,10 +564,10 @@ struct kbase_tlstream;
atom_nr \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_new_atom( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, atom_nr); \
} while (0)
@@ -555,9 +575,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NEW_AS -
* address space object is created
*
- * @kbdev: Kbase device
- * @address_space: Name of the address space object
- * @as_nr: Address space number
+ * @kbdev: Kbase device
+ * @address_space: Name of the address space object
+ * @as_nr: Address space number
*/
#define KBASE_TLSTREAM_TL_NEW_AS( \
kbdev, \
@@ -565,10 +585,10 @@ struct kbase_tlstream;
as_nr \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_new_as( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
address_space, as_nr); \
} while (0)
@@ -576,18 +596,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_DEL_CTX -
* context is destroyed
*
- * @kbdev: Kbase device
- * @ctx: Name of the context object
+ * @kbdev: Kbase device
+ * @ctx: Name of the context object
*/
#define KBASE_TLSTREAM_TL_DEL_CTX( \
kbdev, \
ctx \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_del_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
ctx); \
} while (0)
@@ -595,18 +615,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_DEL_ATOM -
* atom is destroyed
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_DEL_ATOM( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_del_atom( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom); \
} while (0)
@@ -614,9 +634,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU -
* lpu is deleted with gpu
*
- * @kbdev: Kbase device
- * @lpu: Name of the Logical Processing Unit object
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @lpu: Name of the Logical Processing Unit object
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU( \
kbdev, \
@@ -624,10 +644,10 @@ struct kbase_tlstream;
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_lifelink_lpu_gpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
lpu, gpu); \
} while (0)
@@ -635,9 +655,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_LIFELINK_AS_GPU -
* address space is deleted with gpu
*
- * @kbdev: Kbase device
- * @address_space: Name of the address space object
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @address_space: Name of the address space object
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_TL_LIFELINK_AS_GPU( \
kbdev, \
@@ -645,10 +665,10 @@ struct kbase_tlstream;
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_lifelink_as_gpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
address_space, gpu); \
} while (0)
@@ -656,9 +676,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_RET_CTX_LPU -
* context is retained by lpu
*
- * @kbdev: Kbase device
- * @ctx: Name of the context object
- * @lpu: Name of the Logical Processing Unit object
+ * @kbdev: Kbase device
+ * @ctx: Name of the context object
+ * @lpu: Name of the Logical Processing Unit object
*/
#define KBASE_TLSTREAM_TL_RET_CTX_LPU( \
kbdev, \
@@ -666,10 +686,10 @@ struct kbase_tlstream;
lpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_ret_ctx_lpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
ctx, lpu); \
} while (0)
@@ -677,9 +697,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_RET_ATOM_CTX -
* atom is retained by context
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @ctx: Name of the context object
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @ctx: Name of the context object
*/
#define KBASE_TLSTREAM_TL_RET_ATOM_CTX( \
kbdev, \
@@ -687,10 +707,10 @@ struct kbase_tlstream;
ctx \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_ret_atom_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, ctx); \
} while (0)
@@ -698,10 +718,10 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_RET_ATOM_LPU -
* atom is retained by lpu
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @lpu: Name of the Logical Processing Unit object
- * @attrib_match_list: List containing match operator attributes
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @lpu: Name of the Logical Processing Unit object
+ * @attrib_match_list: List containing match operator attributes
*/
#define KBASE_TLSTREAM_TL_RET_ATOM_LPU( \
kbdev, \
@@ -710,10 +730,10 @@ struct kbase_tlstream;
attrib_match_list \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_ret_atom_lpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, lpu, attrib_match_list); \
} while (0)
@@ -721,9 +741,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NRET_CTX_LPU -
* context is released by lpu
*
- * @kbdev: Kbase device
- * @ctx: Name of the context object
- * @lpu: Name of the Logical Processing Unit object
+ * @kbdev: Kbase device
+ * @ctx: Name of the context object
+ * @lpu: Name of the Logical Processing Unit object
*/
#define KBASE_TLSTREAM_TL_NRET_CTX_LPU( \
kbdev, \
@@ -731,10 +751,10 @@ struct kbase_tlstream;
lpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_nret_ctx_lpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
ctx, lpu); \
} while (0)
@@ -742,9 +762,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NRET_ATOM_CTX -
* atom is released by context
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @ctx: Name of the context object
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @ctx: Name of the context object
*/
#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX( \
kbdev, \
@@ -752,10 +772,10 @@ struct kbase_tlstream;
ctx \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_nret_atom_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, ctx); \
} while (0)
@@ -763,9 +783,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NRET_ATOM_LPU -
* atom is released by lpu
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @lpu: Name of the Logical Processing Unit object
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @lpu: Name of the Logical Processing Unit object
*/
#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU( \
kbdev, \
@@ -773,10 +793,10 @@ struct kbase_tlstream;
lpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_nret_atom_lpu( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, lpu); \
} while (0)
@@ -784,9 +804,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_RET_AS_CTX -
* address space is retained by context
*
- * @kbdev: Kbase device
- * @address_space: Name of the address space object
- * @ctx: Name of the context object
+ * @kbdev: Kbase device
+ * @address_space: Name of the address space object
+ * @ctx: Name of the context object
*/
#define KBASE_TLSTREAM_TL_RET_AS_CTX( \
kbdev, \
@@ -794,10 +814,10 @@ struct kbase_tlstream;
ctx \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_ret_as_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
address_space, ctx); \
} while (0)
@@ -805,9 +825,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NRET_AS_CTX -
* address space is released by context
*
- * @kbdev: Kbase device
- * @address_space: Name of the address space object
- * @ctx: Name of the context object
+ * @kbdev: Kbase device
+ * @address_space: Name of the address space object
+ * @ctx: Name of the context object
*/
#define KBASE_TLSTREAM_TL_NRET_AS_CTX( \
kbdev, \
@@ -815,10 +835,10 @@ struct kbase_tlstream;
ctx \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_nret_as_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
address_space, ctx); \
} while (0)
@@ -826,9 +846,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_RET_ATOM_AS -
* atom is retained by address space
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @address_space: Name of the address space object
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @address_space: Name of the address space object
*/
#define KBASE_TLSTREAM_TL_RET_ATOM_AS( \
kbdev, \
@@ -836,10 +856,10 @@ struct kbase_tlstream;
address_space \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_ret_atom_as( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, address_space); \
} while (0)
@@ -847,9 +867,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_NRET_ATOM_AS -
* atom is released by address space
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @address_space: Name of the address space object
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @address_space: Name of the address space object
*/
#define KBASE_TLSTREAM_TL_NRET_ATOM_AS( \
kbdev, \
@@ -857,10 +877,10 @@ struct kbase_tlstream;
address_space \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_nret_atom_as( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, address_space); \
} while (0)
@@ -868,11 +888,11 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG -
* atom job slot attributes
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @descriptor: Job descriptor address
- * @affinity: Job affinity
- * @config: Job config
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @descriptor: Job descriptor address
+ * @affinity: Job affinity
+ * @config: Job config
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG( \
kbdev, \
@@ -882,10 +902,10 @@ struct kbase_tlstream;
config \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_attrib_atom_config( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, descriptor, affinity, config); \
} while (0)
@@ -893,9 +913,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY -
* atom priority
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @prio: Atom priority
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @prio: Atom priority
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY( \
kbdev, \
@@ -903,7 +923,7 @@ struct kbase_tlstream;
prio \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_tl_attrib_atom_priority( \
__TL_DISPATCH_STREAM(kbdev, obj), \
@@ -914,9 +934,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE -
* atom state
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @state: Atom state
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @state: Atom state
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE( \
kbdev, \
@@ -924,7 +944,7 @@ struct kbase_tlstream;
state \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_tl_attrib_atom_state( \
__TL_DISPATCH_STREAM(kbdev, obj), \
@@ -935,15 +955,15 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED -
* atom caused priority change
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_tl_attrib_atom_prioritized( \
__TL_DISPATCH_STREAM(kbdev, obj), \
@@ -954,20 +974,20 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT -
* jit done for atom
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @edit_addr: Address edited by jit
- * @new_addr: Address placed into the edited location
- * @jit_flags: Flags specifying the special requirements for
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @edit_addr: Address edited by jit
+ * @new_addr: Address placed into the edited location
+ * @jit_flags: Flags specifying the special requirements for
* the JIT allocation.
- * @mem_flags: Flags defining the properties of a memory region
- * @j_id: Unique ID provided by the caller, this is used
+ * @mem_flags: Flags defining the properties of a memory region
+ * @j_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests.
- * @com_pgs: The minimum number of physical pages which
+ * @com_pgs: The minimum number of physical pages which
* should back the allocation.
- * @extent: Granularity of physical pages to grow the
+ * @extent: Granularity of physical pages to grow the
* allocation by during a fault.
- * @va_pgs: The minimum number of virtual pages required
+ * @va_pgs: The minimum number of virtual pages required
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \
kbdev, \
@@ -982,7 +1002,7 @@ struct kbase_tlstream;
va_pgs \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \
__kbase_tlstream_tl_attrib_atom_jit( \
__TL_DISPATCH_STREAM(kbdev, obj), \
@@ -993,9 +1013,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_JIT_USEDPAGES -
* used pages for jit
*
- * @kbdev: Kbase device
- * @used_pages: Number of pages used for jit
- * @j_id: Unique ID provided by the caller, this is used
+ * @kbdev: Kbase device
+ * @used_pages: Number of pages used for jit
+ * @j_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests.
*/
#define KBASE_TLSTREAM_TL_JIT_USEDPAGES( \
@@ -1004,10 +1024,10 @@ struct kbase_tlstream;
j_id \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_jit_usedpages( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
used_pages, j_id); \
} while (0)
@@ -1015,22 +1035,22 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO -
* Information about JIT allocations
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @va_pgs: The minimum number of virtual pages required
- * @com_pgs: The minimum number of physical pages which
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @va_pgs: The minimum number of virtual pages required
+ * @com_pgs: The minimum number of physical pages which
* should back the allocation.
- * @extent: Granularity of physical pages to grow the
+ * @extent: Granularity of physical pages to grow the
* allocation by during a fault.
- * @j_id: Unique ID provided by the caller, this is used
+ * @j_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests.
- * @bin_id: The JIT allocation bin, used in conjunction with
+ * @bin_id: The JIT allocation bin, used in conjunction with
* max_allocations to limit the number of each
* type of JIT allocation.
- * @max_allocs: Maximum allocations allowed in this bin.
- * @jit_flags: Flags specifying the special requirements for
+ * @max_allocs: Maximum allocations allowed in this bin.
+ * @jit_flags: Flags specifying the special requirements for
* the JIT allocation.
- * @usg_id: A hint about which allocation should be reused.
+ * @usg_id: A hint about which allocation should be reused.
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( \
kbdev, \
@@ -1045,10 +1065,10 @@ struct kbase_tlstream;
usg_id \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_attrib_atom_jitallocinfo( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, va_pgs, com_pgs, extent, j_id, bin_id, max_allocs, jit_flags, usg_id); \
} while (0)
@@ -1056,9 +1076,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO -
* Information about JIT frees
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
- * @j_id: Unique ID provided by the caller, this is used
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
+ * @j_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests.
*/
#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO( \
@@ -1067,10 +1087,10 @@ struct kbase_tlstream;
j_id \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_attrib_atom_jitfreeinfo( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom, j_id); \
} while (0)
@@ -1078,11 +1098,11 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG -
* address space attributes
*
- * @kbdev: Kbase device
- * @address_space: Name of the address space object
- * @transtab: Configuration of the TRANSTAB register
- * @memattr: Configuration of the MEMATTR register
- * @transcfg: Configuration of the TRANSCFG register (or zero if not present)
+ * @kbdev: Kbase device
+ * @address_space: Name of the address space object
+ * @transtab: Configuration of the TRANSTAB register
+ * @memattr: Configuration of the MEMATTR register
+ * @transcfg: Configuration of the TRANSCFG register (or zero if not present)
*/
#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG( \
kbdev, \
@@ -1092,10 +1112,10 @@ struct kbase_tlstream;
transcfg \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_attrib_as_config( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
address_space, transtab, memattr, transcfg); \
} while (0)
@@ -1103,18 +1123,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP -
* softstop event on given lpu
*
- * @kbdev: Kbase device
- * @lpu: Name of the Logical Processing Unit object
+ * @kbdev: Kbase device
+ * @lpu: Name of the Logical Processing Unit object
*/
#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( \
kbdev, \
lpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_event_lpu_softstop( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
lpu); \
} while (0)
@@ -1122,18 +1142,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX -
* atom softstopped
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_event_atom_softstop_ex( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom); \
} while (0)
@@ -1141,18 +1161,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE -
* atom softstop issued
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_event_atom_softstop_issue( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom); \
} while (0)
@@ -1160,18 +1180,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START -
* atom soft job has started
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_event_atom_softjob_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom); \
} while (0)
@@ -1179,37 +1199,113 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END -
* atom soft job has completed
*
- * @kbdev: Kbase device
- * @atom: Atom identifier
+ * @kbdev: Kbase device
+ * @atom: Atom identifier
*/
#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END( \
kbdev, \
atom \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_tl_event_atom_softjob_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
atom); \
} while (0)
/**
+ * KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED -
+ * Arbiter has granted gpu access
+ *
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
+ */
+#define KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED( \
+ kbdev, \
+ gpu \
+ ) \
+ do { \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
+ if (enabled & TLSTREAM_ENABLED) \
+ __kbase_tlstream_tl_event_arb_granted( \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
+ gpu); \
+ } while (0)
+
+/**
+ * KBASE_TLSTREAM_TL_EVENT_ARB_STARTED -
+ * Driver is running again and able to process jobs
+ *
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
+ */
+#define KBASE_TLSTREAM_TL_EVENT_ARB_STARTED( \
+ kbdev, \
+ gpu \
+ ) \
+ do { \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
+ if (enabled & TLSTREAM_ENABLED) \
+ __kbase_tlstream_tl_event_arb_started( \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
+ gpu); \
+ } while (0)
+
+/**
+ * KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED -
+ * Arbiter has requested driver to stop using gpu
+ *
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
+ */
+#define KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED( \
+ kbdev, \
+ gpu \
+ ) \
+ do { \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
+ if (enabled & TLSTREAM_ENABLED) \
+ __kbase_tlstream_tl_event_arb_stop_requested( \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
+ gpu); \
+ } while (0)
+
+/**
+ * KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED -
+ * Driver has stopped using gpu
+ *
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
+ */
+#define KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED( \
+ kbdev, \
+ gpu \
+ ) \
+ do { \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
+ if (enabled & TLSTREAM_ENABLED) \
+ __kbase_tlstream_tl_event_arb_stopped( \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
+ gpu); \
+ } while (0)
+
+/**
* KBASE_TLSTREAM_JD_GPU_SOFT_RESET -
* gpu soft reset
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET( \
kbdev, \
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_jd_gpu_soft_reset( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
+ __TL_DISPATCH_STREAM(kbdev, obj), \
gpu); \
} while (0)
@@ -1217,9 +1313,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PM_STATE -
* PM state
*
- * @kbdev: Kbase device
- * @core_type: Core type (shader, tiler, l2 cache, l3 cache)
- * @core_state_bitset: 64bits bitmask reporting power state of the cores
+ * @kbdev: Kbase device
+ * @core_type: Core type (shader, tiler, l2 cache, l3 cache)
+ * @core_state_bitset: 64bits bitmask reporting power state of the cores
* (1-ON, 0-OFF)
*/
#define KBASE_TLSTREAM_AUX_PM_STATE( \
@@ -1228,10 +1324,10 @@ struct kbase_tlstream;
core_state_bitset \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_pm_state( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
core_type, core_state_bitset); \
} while (0)
@@ -1239,10 +1335,10 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PAGEFAULT -
* Page fault
*
- * @kbdev: Kbase device
- * @ctx_nr: Kernel context number
- * @as_nr: Address space number
- * @page_cnt_change: Number of pages to be added
+ * @kbdev: Kbase device
+ * @ctx_nr: Kernel context number
+ * @as_nr: Address space number
+ * @page_cnt_change: Number of pages to be added
*/
#define KBASE_TLSTREAM_AUX_PAGEFAULT( \
kbdev, \
@@ -1251,10 +1347,10 @@ struct kbase_tlstream;
page_cnt_change \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_pagefault( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
ctx_nr, as_nr, page_cnt_change); \
} while (0)
@@ -1262,9 +1358,9 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PAGESALLOC -
* Total alloc pages change
*
- * @kbdev: Kbase device
- * @ctx_nr: Kernel context number
- * @page_cnt: Number of pages used by the context
+ * @kbdev: Kbase device
+ * @ctx_nr: Kernel context number
+ * @page_cnt: Number of pages used by the context
*/
#define KBASE_TLSTREAM_AUX_PAGESALLOC( \
kbdev, \
@@ -1272,10 +1368,10 @@ struct kbase_tlstream;
page_cnt \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_pagesalloc( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
ctx_nr, page_cnt); \
} while (0)
@@ -1283,18 +1379,18 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_DEVFREQ_TARGET -
* New device frequency target
*
- * @kbdev: Kbase device
- * @target_freq: New target frequency
+ * @kbdev: Kbase device
+ * @target_freq: New target frequency
*/
#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET( \
kbdev, \
target_freq \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_devfreq_target( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
target_freq); \
} while (0)
@@ -1302,15 +1398,15 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START -
* enter protected mode start
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START( \
kbdev, \
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_aux_protected_enter_start( \
__TL_DISPATCH_STREAM(kbdev, aux), \
@@ -1321,15 +1417,15 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END -
* enter protected mode end
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END( \
kbdev, \
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_aux_protected_enter_end( \
__TL_DISPATCH_STREAM(kbdev, aux), \
@@ -1340,15 +1436,15 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START -
* leave protected mode start
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START( \
kbdev, \
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_aux_protected_leave_start( \
__TL_DISPATCH_STREAM(kbdev, aux), \
@@ -1359,15 +1455,15 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END -
* leave protected mode end
*
- * @kbdev: Kbase device
- * @gpu: Name of the GPU object
+ * @kbdev: Kbase device
+ * @gpu: Name of the GPU object
*/
#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END( \
kbdev, \
gpu \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \
__kbase_tlstream_aux_protected_leave_end( \
__TL_DISPATCH_STREAM(kbdev, aux), \
@@ -1378,13 +1474,13 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_JIT_STATS -
* per-bin JIT statistics
*
- * @kbdev: Kbase device
- * @ctx_nr: Kernel context number
- * @bid: JIT bin id
- * @max_allocs: Maximum allocations allowed in this bin.
- * @allocs: Number of active allocations in this bin
- * @va_pages: Number of virtual pages allocated in this bin
- * @ph_pages: Number of physical pages allocated in this bin
+ * @kbdev: Kbase device
+ * @ctx_nr: Kernel context number
+ * @bid: JIT bin id
+ * @max_allocs: Maximum allocations allowed in this bin.
+ * @allocs: Number of active allocations in this bin
+ * @va_pages: Number of virtual pages allocated in this bin
+ * @ph_pages: Number of physical pages allocated in this bin
*/
#define KBASE_TLSTREAM_AUX_JIT_STATS( \
kbdev, \
@@ -1396,10 +1492,10 @@ struct kbase_tlstream;
ph_pages \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_jit_stats( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
ctx_nr, bid, max_allocs, allocs, va_pages, ph_pages); \
} while (0)
@@ -1407,11 +1503,11 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT -
* event on a given job slot
*
- * @kbdev: Kbase device
- * @ctx: Name of the context object
- * @slot_nr: Job slot number
- * @atom_nr: Sequential number of an atom
- * @event: Event type. One of TL_JS_EVENT values
+ * @kbdev: Kbase device
+ * @ctx: Name of the context object
+ * @slot_nr: Job slot number
+ * @atom_nr: Sequential number of an atom
+ * @event: Event type. One of TL_JS_EVENT values
*/
#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( \
kbdev, \
@@ -1421,10 +1517,10 @@ struct kbase_tlstream;
event \
) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
if (enabled & TLSTREAM_ENABLED) \
__kbase_tlstream_aux_event_job_slot( \
- __TL_DISPATCH_STREAM(kbdev, aux), \
+ __TL_DISPATCH_STREAM(kbdev, aux), \
ctx, slot_nr, atom_nr, event); \
} while (0)
@@ -1432,33 +1528,29 @@ struct kbase_tlstream;
* KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE -
* New KBase Device
*
- * @kbdev: Kbase device
- * @kbase_device_id: The id of the physical hardware
- * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware
- * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports
+ * @kbdev: Kbase device
+ * @kbase_device_id: The id of the physical hardware
+ * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware
+ * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports
+ * @kbase_device_as_count: The number of address spaces the physical hardware has available
*/
#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \
kbdev, \
kbase_device_id, \
kbase_device_gpu_core_count, \
- kbase_device_max_num_csgs \
+ kbase_device_max_num_csgs, \
+ kbase_device_as_count \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_new_device( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kbase_device_id, kbase_device_gpu_core_count, kbase_device_max_num_csgs); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG -
* CSG is programmed to a slot
*
- * @kbdev: Kbase device
- * @kbase_device_id: The id of the physical hardware
- * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace
- * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed
+ * @kbdev: Kbase device
+ * @kbase_device_id: The id of the physical hardware
+ * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace
+ * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed
*/
#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \
kbdev, \
@@ -1466,83 +1558,87 @@ struct kbase_tlstream;
gpu_cmdq_grp_handle, \
kbase_device_csg_slot_index \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_device_program_csg( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kbase_device_id, gpu_cmdq_grp_handle, kbase_device_csg_slot_index); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG -
* CSG is deprogrammed from a slot
*
- * @kbdev: Kbase device
- * @kbase_device_id: The id of the physical hardware
- * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed
+ * @kbdev: Kbase device
+ * @kbase_device_id: The id of the physical hardware
+ * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed
*/
#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \
kbdev, \
kbase_device_id, \
kbase_device_csg_slot_index \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_device_deprogram_csg( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kbase_device_id, kbase_device_csg_slot_index); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_NEW_CTX -
* New KBase Context
*
- * @kbdev: Kbase device
- * @kernel_ctx_id: Unique ID for the KBase Context
- * @kbase_device_id: The id of the physical hardware
+ * @kbdev: Kbase device
+ * @kernel_ctx_id: Unique ID for the KBase Context
+ * @kbase_device_id: The id of the physical hardware
*/
#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \
kbdev, \
kernel_ctx_id, \
kbase_device_id \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_new_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kernel_ctx_id, kbase_device_id); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_DEL_CTX -
* Delete KBase Context
*
- * @kbdev: Kbase device
- * @kernel_ctx_id: Unique ID for the KBase Context
+ * @kbdev: Kbase device
+ * @kernel_ctx_id: Unique ID for the KBase Context
*/
#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \
kbdev, \
kernel_ctx_id \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_del_ctx( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kernel_ctx_id); \
- } while (0)
+ do { } while (0)
+
+/**
+ * KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS -
+ * Address Space is assigned to a KBase context
+ *
+ * @kbdev: Kbase device
+ * @kernel_ctx_id: Unique ID for the KBase Context
+ * @kbase_device_as_index: The index of the device address space being assigned
+ */
+#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \
+ kbdev, \
+ kernel_ctx_id, \
+ kbase_device_as_index \
+ ) \
+ do { } while (0)
+
+/**
+ * KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS -
+ * Address Space is unassigned from a KBase context
+ *
+ * @kbdev: Kbase device
+ * @kernel_ctx_id: Unique ID for the KBase Context
+ */
+#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \
+ kbdev, \
+ kernel_ctx_id \
+ ) \
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE -
* New KCPU Queue
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @kernel_ctx_id: Unique ID for the KBase Context
- * @kcpuq_num_pending_cmds: Number of commands already enqueued
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @kernel_ctx_id: Unique ID for the KBase Context
+ * @kcpuq_num_pending_cmds: Number of commands already enqueued
* in the KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \
@@ -1551,102 +1647,72 @@ struct kbase_tlstream;
kernel_ctx_id, \
kcpuq_num_pending_cmds \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_new_kcpuqueue( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, kernel_ctx_id, kcpuq_num_pending_cmds); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE -
* Delete KCPU Queue
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_del_kcpuqueue( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL -
* KCPU Queue enqueues Signal on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @fence: Fence object handle
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @fence: Fence object handle
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \
kbdev, \
kcpu_queue, \
fence \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, fence); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT -
* KCPU Queue enqueues Wait on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @fence: Fence object handle
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @fence: Fence object handle
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \
kbdev, \
kcpu_queue, \
fence \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, fence); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT -
* Begin array of KCPU Queue enqueues Wait on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_wait( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT -
* Array item of KCPU Queue enqueues Wait on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @cqs_obj_gpu_addr: CQS Object GPU ptr
- * @cqs_obj_compare_value: Semaphore value that should be exceeded
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @cqs_obj_gpu_addr: CQS Object GPU ptr
+ * @cqs_obj_compare_value: Semaphore value that should be exceeded
* for the WAIT to pass
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT( \
@@ -1655,256 +1721,184 @@ struct kbase_tlstream;
cqs_obj_gpu_addr, \
cqs_obj_compare_value \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_wait( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, cqs_obj_gpu_addr, cqs_obj_compare_value); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT -
* End array of KCPU Queue enqueues Wait on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_wait( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET -
* Begin array of KCPU Queue enqueues Set on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_set( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET -
* Array item of KCPU Queue enqueues Set on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @cqs_obj_gpu_addr: CQS Object GPU ptr
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @cqs_obj_gpu_addr: CQS Object GPU ptr
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET( \
kbdev, \
kcpu_queue, \
cqs_obj_gpu_addr \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_set( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, cqs_obj_gpu_addr); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET -
* End array of KCPU Queue enqueues Set on Cross Queue Sync Object
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_set( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY -
* Begin array of KCPU Queue enqueues Debug Copy
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_debugcopy( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY -
* Array item of KCPU Queue enqueues Debug Copy
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @debugcopy_dst_size: Debug Copy destination size
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @debugcopy_dst_size: Debug Copy destination size
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \
kbdev, \
kcpu_queue, \
debugcopy_dst_size \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_debugcopy( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, debugcopy_dst_size); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY -
* End array of KCPU Queue enqueues Debug Copy
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_debugcopy( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT -
* KCPU Queue enqueues Map Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @map_import_buf_gpu_addr: Map import buffer GPU ptr
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @map_import_buf_gpu_addr: Map import buffer GPU ptr
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \
kbdev, \
kcpu_queue, \
map_import_buf_gpu_addr \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, map_import_buf_gpu_addr); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT -
* KCPU Queue enqueues Unmap Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @map_import_buf_gpu_addr: Map import buffer GPU ptr
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @map_import_buf_gpu_addr: Map import buffer GPU ptr
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \
kbdev, \
kcpu_queue, \
map_import_buf_gpu_addr \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, map_import_buf_gpu_addr); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE -
* KCPU Queue enqueues Unmap Import ignoring reference count
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @map_import_buf_gpu_addr: Map import buffer GPU ptr
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @map_import_buf_gpu_addr: Map import buffer GPU ptr
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \
kbdev, \
kcpu_queue, \
map_import_buf_gpu_addr \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, map_import_buf_gpu_addr); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC -
* Begin array of KCPU Queue enqueues JIT Alloc
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC -
* Array item of KCPU Queue enqueues JIT Alloc
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write
* the JIT allocated GPU virtual address to
- * @jit_alloc_va_pages: The minimum number of virtual pages required
- * @jit_alloc_commit_pages: The minimum number of physical pages which
+ * @jit_alloc_va_pages: The minimum number of virtual pages required
+ * @jit_alloc_commit_pages: The minimum number of physical pages which
* should back the allocation
- * @jit_alloc_extent: Granularity of physical pages to grow the allocation
+ * @jit_alloc_extent: Granularity of physical pages to grow the allocation
* by during a fault
- * @jit_alloc_jit_id: Unique ID provided by the caller, this is used
+ * @jit_alloc_jit_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests. Zero is not a valid value
- * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with
+ * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with
* max_allocations to limit the number of each type of JIT allocation
- * @jit_alloc_max_allocations: The maximum number of allocations
+ * @jit_alloc_max_allocations: The maximum number of allocations
* allowed within the bin specified by bin_id. Should be the same for all
* JIT allocations within the same bin.
- * @jit_alloc_flags: Flags specifying the special requirements for the
+ * @jit_alloc_flags: Flags specifying the special requirements for the
* JIT allocation
- * @jit_alloc_usage_id: A hint about which allocation should be
+ * @jit_alloc_usage_id: A hint about which allocation should be
* reused. The kernel should attempt to use a previous allocation with the same
* usage_id
*/
@@ -1921,59 +1915,41 @@ struct kbase_tlstream;
jit_alloc_flags, \
jit_alloc_usage_id \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, jit_alloc_gpu_alloc_addr_dest, jit_alloc_va_pages, jit_alloc_commit_pages, jit_alloc_extent, jit_alloc_jit_id, jit_alloc_bin_id, jit_alloc_max_allocations, jit_alloc_flags, jit_alloc_usage_id); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC -
* End array of KCPU Queue enqueues JIT Alloc
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE -
* Begin array of KCPU Queue enqueues JIT Free
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE -
* Array item of KCPU Queue enqueues JIT Free
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @jit_alloc_jit_id: Unique ID provided by the caller, this is used
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @jit_alloc_jit_id: Unique ID provided by the caller, this is used
* to pair allocation and free requests. Zero is not a valid value
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \
@@ -1981,364 +1957,250 @@ struct kbase_tlstream;
kcpu_queue, \
jit_alloc_jit_id \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, jit_alloc_jit_id); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE -
* End array of KCPU Queue enqueues JIT Free
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START -
* KCPU Queue starts a Signal on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END -
* KCPU Queue ends a Signal on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START -
* KCPU Queue starts a Wait on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END -
* KCPU Queue ends a Wait on Fence
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START -
* KCPU Queue starts a Wait on an array of Cross Queue Sync Objects
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END -
* KCPU Queue ends a Wait on an array of Cross Queue Sync Objects
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET -
* KCPU Queue executes a Set on an array of Cross Queue Sync Objects
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START -
* KCPU Queue starts an array of Debug Copys
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END -
* KCPU Queue ends an array of Debug Copys
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START -
* KCPU Queue starts a Map Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END -
* KCPU Queue ends a Map Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START -
* KCPU Queue starts an Unmap Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END -
* KCPU Queue ends an Unmap Import
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START -
* KCPU Queue starts an Unmap Import ignoring reference count
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END -
* KCPU Queue ends an Unmap Import ignoring reference count
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START -
* KCPU Queue starts an array of JIT Allocs
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END -
* Begin array of KCPU Queue ends an array of JIT Allocs
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END -
* Array item of KCPU Queue ends an array of JIT Allocs
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address
- * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address
+ * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \
kbdev, \
@@ -2346,78 +2208,54 @@ struct kbase_tlstream;
jit_alloc_gpu_alloc_addr, \
jit_alloc_mmu_flags \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, jit_alloc_gpu_alloc_addr, jit_alloc_mmu_flags); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END -
* End array of KCPU Queue ends an array of JIT Allocs
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START -
* KCPU Queue starts an array of JIT Frees
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END -
* Begin array of KCPU Queue ends an array of JIT Frees
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END -
* Array item of KCPU Queue ends an array of JIT Frees
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
- * @jit_free_pages_used: The actual number of pages used by the JIT
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
+ * @jit_free_pages_used: The actual number of pages used by the JIT
* allocation
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \
@@ -2425,72 +2263,48 @@ struct kbase_tlstream;
kcpu_queue, \
jit_free_pages_used \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue, jit_free_pages_used); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END -
* End array of KCPU Queue ends an array of JIT Frees
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER -
* KCPU Queue executes an Error Barrier
*
- * @kbdev: Kbase device
- * @kcpu_queue: KCPU queue
+ * @kbdev: Kbase device
+ * @kcpu_queue: KCPU queue
*/
#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \
kbdev, \
kcpu_queue \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & TLSTREAM_ENABLED) \
- __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- kcpu_queue); \
- } while (0)
+ do { } while (0)
/**
* KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW -
* An overflow has happened with the CSFFW Timeline stream
*
- * @kbdev: Kbase device
- * @csffw_timestamp: Timestamp of a CSFFW event
- * @csffw_cycle: Cycle number of a CSFFW event
+ * @kbdev: Kbase device
+ * @csffw_timestamp: Timestamp of a CSFFW event
+ * @csffw_cycle: Cycle number of a CSFFW event
*/
#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \
kbdev, \
csffw_timestamp, \
csffw_cycle \
) \
- do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \
- __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \
- __TL_DISPATCH_STREAM(kbdev, obj), \
- csffw_timestamp, csffw_cycle); \
- } while (0)
+ do { } while (0)
/* Gator tracepoints are hooked into TLSTREAM interface.
@@ -2504,10 +2318,10 @@ struct kbase_tlstream;
* with corresponding GATOR_JOB_SLOT values.
*/
#undef KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT
-#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \
+#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \
context, slot_nr, atom_nr, event) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
kbase_trace_mali_job_slots_event(kbdev->id, \
GATOR_MAKE_EVENT(event, slot_nr), \
context, (u8) atom_nr); \
@@ -2520,7 +2334,7 @@ struct kbase_tlstream;
#undef KBASE_TLSTREAM_AUX_PM_STATE
#define KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
kbase_trace_mali_pm_status(kbdev->id, \
core_type, state); \
if (enabled & TLSTREAM_ENABLED) \
@@ -2530,10 +2344,10 @@ struct kbase_tlstream;
} while (0)
#undef KBASE_TLSTREAM_AUX_PAGEFAULT
-#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \
+#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \
ctx_nr, as_nr, page_cnt_change) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
kbase_trace_mali_page_fault_insert_pages(kbdev->id, \
as_nr, \
page_cnt_change); \
@@ -2550,8 +2364,8 @@ struct kbase_tlstream;
#undef KBASE_TLSTREAM_AUX_PAGESALLOC
#define KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, ctx_nr, page_cnt) \
do { \
- int enabled = atomic_read(&kbdev->timeline_is_enabled); \
- u32 global_pages_count = \
+ int enabled = atomic_read(&kbdev->timeline_flags); \
+ u32 global_pages_count = \
atomic_read(&kbdev->memdev.used_pages); \
\
kbase_trace_mali_total_alloc_pages_change(kbdev->id, \