From 6251d38059ae22304ede4f3748af9f795bdbf4fd Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Wed, 28 Sep 2022 19:28:34 -0500
Subject: [PATCH 001/151] ACPI: ARM Performance Monitoring Unit Table (APMT)
initial support
ARM Performance Monitoring Unit Table describes the properties of PMU
support in ARM-based system. The APMT table contains a list of nodes,
each represents a PMU in the system that conforms to ARM CoreSight PMU
architecture. The properties of each node include information required
to access the PMU (e.g. MMIO base address, interrupt number) and also
identification. For more detailed information, please refer to the
specification below:
* APMT: https://developer.arm.com/documentation/den0117/latest
* ARM Coresight PMU:
https://developer.arm.com/documentation/ihi0091/latest
The initial support adds the detection of APMT table and generic
infrastructure to create platform devices for ARM CoreSight PMUs.
Similar to IORT the root pointer of APMT is preserved during runtime
and each PMU platform device is given a pointer to the corresponding
APMT node.
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Acked-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reviewed-by: Sudeep Holla <sudeep.holla@arm.com>
Link: https://lore.kernel.org/r/20220929002834.32664-1-bwicaksono@nvidia.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 1 +
drivers/acpi/arm64/Kconfig | 3 +
drivers/acpi/arm64/Makefile | 1 +
drivers/acpi/arm64/apmt.c | 177 ++++++++++++++++++++++++++++++++++++
drivers/acpi/bus.c | 2 +
include/linux/acpi_apmt.h | 19 ++++
6 files changed, 203 insertions(+)
create mode 100644 drivers/acpi/arm64/apmt.c
create mode 100644 include/linux/acpi_apmt.h
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 505c8a1ccbe0..16c2a7d68ca4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
config ARM64
def_bool y
+ select ACPI_APMT if ACPI
select ACPI_CCA_REQUIRED if ACPI
select ACPI_GENERIC_GSI if ACPI
select ACPI_GTDT if ACPI
diff --git a/drivers/acpi/arm64/Kconfig b/drivers/acpi/arm64/Kconfig
index d4a72835f328..b3ed6212244c 100644
--- a/drivers/acpi/arm64/Kconfig
+++ b/drivers/acpi/arm64/Kconfig
@@ -18,3 +18,6 @@ config ACPI_AGDI
reset command.
If set, the kernel parses AGDI table and listens for the command.
+
+config ACPI_APMT
+ bool
diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile
index 7b9e4045659d..e21a9e84e394 100644
--- a/drivers/acpi/arm64/Makefile
+++ b/drivers/acpi/arm64/Makefile
@@ -2,4 +2,5 @@
obj-$(CONFIG_ACPI_AGDI) += agdi.o
obj-$(CONFIG_ACPI_IORT) += iort.o
obj-$(CONFIG_ACPI_GTDT) += gtdt.o
+obj-$(CONFIG_ACPI_APMT) += apmt.o
obj-y += dma.o
diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c
new file mode 100644
index 000000000000..f55167ca51e7
--- /dev/null
+++ b/drivers/acpi/arm64/apmt.c
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ARM APMT table support.
+ * Design document number: ARM DEN0117.
+ *
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
+ *
+ */
+
+#define pr_fmt(fmt) "ACPI: APMT: " fmt
+
+#include <linux/acpi.h>
+#include <linux/acpi_apmt.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+
+#define DEV_NAME "arm-cs-arch-pmu"
+
+/* There can be up to 3 resources: page 0 and 1 address, and interrupt. */
+#define DEV_MAX_RESOURCE_COUNT 3
+
+/* Root pointer to the mapped APMT table */
+static struct acpi_table_header *apmt_table;
+
+static int __init apmt_init_resources(struct resource *res,
+ struct acpi_apmt_node *node)
+{
+ int irq, trigger;
+ int num_res = 0;
+
+ res[num_res].start = node->base_address0;
+ res[num_res].end = node->base_address0 + SZ_4K - 1;
+ res[num_res].flags = IORESOURCE_MEM;
+
+ num_res++;
+
+ res[num_res].start = node->base_address1;
+ res[num_res].end = node->base_address1 + SZ_4K - 1;
+ res[num_res].flags = IORESOURCE_MEM;
+
+ num_res++;
+
+ if (node->ovflw_irq != 0) {
+ trigger = (node->ovflw_irq_flags & ACPI_APMT_OVFLW_IRQ_FLAGS_MODE);
+ trigger = (trigger == ACPI_APMT_OVFLW_IRQ_FLAGS_MODE_LEVEL) ?
+ ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE;
+ irq = acpi_register_gsi(NULL, node->ovflw_irq, trigger,
+ ACPI_ACTIVE_HIGH);
+
+ if (irq <= 0) {
+ pr_warn("APMT could not register gsi hwirq %d\n", irq);
+ return num_res;
+ }
+
+ res[num_res].start = irq;
+ res[num_res].end = irq;
+ res[num_res].flags = IORESOURCE_IRQ;
+
+ num_res++;
+ }
+
+ return num_res;
+}
+
+/**
+ * apmt_add_platform_device() - Allocate a platform device for APMT node
+ * @node: Pointer to device ACPI APMT node
+ *
+ * Returns: 0 on success, <0 failure
+ */
+static int __init apmt_add_platform_device(struct acpi_apmt_node *node,
+ struct fwnode_handle *fwnode)
+{
+ struct platform_device *pdev;
+ int ret, count;
+ struct resource res[DEV_MAX_RESOURCE_COUNT];
+
+ pdev = platform_device_alloc(DEV_NAME, PLATFORM_DEVID_AUTO);
+ if (!pdev)
+ return -ENOMEM;
+
+ memset(res, 0, sizeof(res));
+
+ count = apmt_init_resources(res, node);
+
+ ret = platform_device_add_resources(pdev, res, count);
+ if (ret)
+ goto dev_put;
+
+ /*
+ * Add a copy of APMT node pointer to platform_data to be used to
+ * retrieve APMT data information.
+ */
+ ret = platform_device_add_data(pdev, &node, sizeof(node));
+ if (ret)
+ goto dev_put;
+
+ pdev->dev.fwnode = fwnode;
+
+ ret = platform_device_add(pdev);
+
+ if (ret)
+ goto dev_put;
+
+ return 0;
+
+dev_put:
+ platform_device_put(pdev);
+
+ return ret;
+}
+
+static int __init apmt_init_platform_devices(void)
+{
+ struct acpi_apmt_node *apmt_node;
+ struct acpi_table_apmt *apmt;
+ struct fwnode_handle *fwnode;
+ u64 offset, end;
+ int ret;
+
+ /*
+ * apmt_table and apmt both point to the start of APMT table, but
+ * have different struct types
+ */
+ apmt = (struct acpi_table_apmt *)apmt_table;
+ offset = sizeof(*apmt);
+ end = apmt->header.length;
+
+ while (offset < end) {
+ apmt_node = ACPI_ADD_PTR(struct acpi_apmt_node, apmt,
+ offset);
+
+ fwnode = acpi_alloc_fwnode_static();
+ if (!fwnode)
+ return -ENOMEM;
+
+ ret = apmt_add_platform_device(apmt_node, fwnode);
+ if (ret) {
+ acpi_free_fwnode_static(fwnode);
+ return ret;
+ }
+
+ offset += apmt_node->length;
+ }
+
+ return 0;
+}
+
+void __init acpi_apmt_init(void)
+{
+ acpi_status status;
+ int ret;
+
+ /**
+ * APMT table nodes will be used at runtime after the apmt init,
+ * so we don't need to call acpi_put_table() to release
+ * the APMT table mapping.
+ */
+ status = acpi_get_table(ACPI_SIG_APMT, 0, &apmt_table);
+
+ if (ACPI_FAILURE(status)) {
+ if (status != AE_NOT_FOUND) {
+ const char *msg = acpi_format_exception(status);
+
+ pr_err("Failed to get APMT table, %s\n", msg);
+ }
+
+ return;
+ }
+
+ ret = apmt_init_platform_devices();
+ if (ret) {
+ pr_err("Failed to initialize APMT platform devices, ret: %d\n", ret);
+ acpi_put_table(apmt_table);
+ }
+}
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index d466c8195314..351208eda9be 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -27,6 +27,7 @@
#include <linux/dmi.h>
#endif
#include <linux/acpi_agdi.h>
+#include <linux/acpi_apmt.h>
#include <linux/acpi_iort.h>
#include <linux/acpi_viot.h>
#include <linux/pci.h>
@@ -1423,6 +1424,7 @@ static int __init acpi_init(void)
acpi_setup_sb_notify_handler();
acpi_viot_init();
acpi_agdi_init();
+ acpi_apmt_init();
return 0;
}
diff --git a/include/linux/acpi_apmt.h b/include/linux/acpi_apmt.h
new file mode 100644
index 000000000000..40bd634d082f
--- /dev/null
+++ b/include/linux/acpi_apmt.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * ARM CoreSight PMU driver.
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
+ *
+ */
+
+#ifndef __ACPI_APMT_H__
+#define __ACPI_APMT_H__
+
+#include <linux/acpi.h>
+
+#ifdef CONFIG_ACPI_APMT
+void acpi_apmt_init(void);
+#else
+static inline void acpi_apmt_init(void) { }
+#endif /* CONFIG_ACPI_APMT */
+
+#endif /* __ACPI_APMT_H__ */
From 05da178ce0aa152f0592e10fa874054187f3621b Mon Sep 17 00:00:00 2001
From: Robin Murphy <robin.murphy@arm.com>
Date: Wed, 28 Sep 2022 20:21:26 +0100
Subject: [PATCH 002/151] ACPI/IORT: Update SMMUv3 DeviceID support
IORT E.e now allows SMMUv3 nodes to describe the DeviceID for MSIs
independently of wired GSIVs, where the previous oddly-restrictive
definition meant that an SMMU without PRI support had to provide a
DeviceID even if it didn't support MSIs either. Support this, with
the usual temporary flag definition while the real one is making
its way through ACPICA.
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Link: https://lore.kernel.org/r/4b3e2ead4f392d1a47a7528da119d57918e5d806.1664392886.git.robin.murphy@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/acpi/arm64/iort.c | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index 8059baf4ef27..38fb84974f35 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -402,6 +402,10 @@ static struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
return NULL;
}
+#ifndef ACPI_IORT_SMMU_V3_DEVICEID_VALID
+#define ACPI_IORT_SMMU_V3_DEVICEID_VALID (1 << 4)
+#endif
+
static int iort_get_id_mapping_index(struct acpi_iort_node *node)
{
struct acpi_iort_smmu_v3 *smmu;
@@ -418,12 +422,16 @@ static int iort_get_id_mapping_index(struct acpi_iort_node *node)
smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
/*
- * ID mapping index is only ignored if all interrupts are
- * GSIV based
+ * Until IORT E.e (node rev. 5), the ID mapping index was
+ * defined to be valid unless all interrupts are GSIV-based.
*/
- if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
- && smmu->sync_gsiv)
+ if (node->revision < 5) {
+ if (smmu->event_gsiv && smmu->pri_gsiv &&
+ smmu->gerr_gsiv && smmu->sync_gsiv)
+ return -EINVAL;
+ } else if (!(smmu->flags & ACPI_IORT_SMMU_V3_DEVICEID_VALID)) {
return -EINVAL;
+ }
if (smmu->id_mapping_index >= node->mapping_count) {
pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",
From efbc95787cbeaaad66cde36e6acd2535bf036d56 Mon Sep 17 00:00:00 2001
From: Jisheng Zhang <jszhang@kernel.org>
Date: Thu, 6 Oct 2022 15:55:41 +0800
Subject: [PATCH 003/151] arm64: jump_label: mark arguments as const to satisfy
asm constraints
Inspired by x86 commit 864b435514b2("x86/jump_label: Mark arguments as
const to satisfy asm constraints"), mark arch_static_branch()'s and
arch_static_branch_jump()'s arguments as const to satisfy asm
constraints. And Steven in [1] also pointed out that "The "i"
constraint needs to be a constant."
Tested with building a simple external kernel module with "O0".
[1]https://lore.kernel.org/all/20210212094059.5f8d05e8@gandalf.local.home/
Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Link: https://lore.kernel.org/r/20221006075542.2658-2-jszhang@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/jump_label.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
index cea441b6aa5d..48ddc0f45d22 100644
--- a/arch/arm64/include/asm/jump_label.h
+++ b/arch/arm64/include/asm/jump_label.h
@@ -15,8 +15,8 @@
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
-static __always_inline bool arch_static_branch(struct static_key *key,
- bool branch)
+static __always_inline bool arch_static_branch(struct static_key * const key,
+ const bool branch)
{
asm_volatile_goto(
"1: nop \n\t"
@@ -32,8 +32,8 @@ l_yes:
return true;
}
-static __always_inline bool arch_static_branch_jump(struct static_key *key,
- bool branch)
+static __always_inline bool arch_static_branch_jump(struct static_key * const key,
+ const bool branch)
{
asm_volatile_goto(
"1: b %l[l_yes] \n\t"
From b9024f87d63fde052daa8687d16a23ea0f894bb0 Mon Sep 17 00:00:00 2001
From: Jisheng Zhang <jszhang@kernel.org>
Date: Thu, 6 Oct 2022 15:55:42 +0800
Subject: [PATCH 004/151] arm64: alternative: constify
alternative_has_feature_* argument
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Inspired by x86 commit 864b435514b2("x86/jump_label: Mark arguments as
const to satisfy asm constraints"), constify alternative_has_feature_*
argument to satisfy asm constraints. And Steven in [1] also pointed
out that "The "i" constraint needs to be a constant."
Tested with building a simple external kernel module with "O0".
Before the patch, got similar gcc warnings and errors as below:
In file included from <command-line>:
In function ‘alternative_has_feature_likely’,
inlined from ‘system_capabilities_finalized’ at
arch/arm64/include/asm/cpufeature.h:440:9,
inlined from ‘arm64_preempt_schedule_irq’ at
arch/arm64/kernel/entry-common.c:264:6:
include/linux/compiler_types.h:285:33: warning:
‘asm’ operand 0 probably does not match constraints
285 | #define asm_volatile_goto(x...) asm goto(x)
| ^~~
arch/arm64/include/asm/alternative-macros.h:232:9:
note: in expansion of macro ‘asm_volatile_goto’
232 | asm_volatile_goto(
| ^~~~~~~~~~~~~~~~~
include/linux/compiler_types.h:285:33: error:
impossible constraint in ‘asm’
285 | #define asm_volatile_goto(x...) asm goto(x)
| ^~~
arch/arm64/include/asm/alternative-macros.h:232:9:
note: in expansion of macro ‘asm_volatile_goto’
232 | asm_volatile_goto(
| ^~~~~~~~~~~~~~~~~
After the patch, the simple external test kernel module is built fine
with "-O0".
[1]https://lore.kernel.org/all/20210212094059.5f8d05e8@gandalf.local.home/
Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Link: https://lore.kernel.org/r/20221006075542.2658-3-jszhang@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/alternative-macros.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h
index 3622e9f4fb44..bdf1f6bcd010 100644
--- a/arch/arm64/include/asm/alternative-macros.h
+++ b/arch/arm64/include/asm/alternative-macros.h
@@ -224,7 +224,7 @@ alternative_endif
#include <linux/types.h>
static __always_inline bool
-alternative_has_feature_likely(unsigned long feature)
+alternative_has_feature_likely(const unsigned long feature)
{
compiletime_assert(feature < ARM64_NCAPS,
"feature must be < ARM64_NCAPS");
@@ -242,7 +242,7 @@ l_no:
}
static __always_inline bool
-alternative_has_feature_unlikely(unsigned long feature)
+alternative_has_feature_unlikely(const unsigned long feature)
{
compiletime_assert(feature < ARM64_NCAPS,
"feature must be < ARM64_NCAPS");
From 994b7ac1697b4581b7726d2ac64321e3c840229b Mon Sep 17 00:00:00 2001
From: Masahiro Yamada <masahiroy@kernel.org>
Date: Thu, 13 Oct 2022 08:35:00 +0900
Subject: [PATCH 005/151] arm64: remove special treatment for the link order of
head.o
In the previous discussion (see the Link tag), Ard pointed out that
arm/arm64/kernel/head.o does not need any special treatment - the only
piece that must appear right at the start of the binary image is the
image header which is emitted into .head.text.
The linker script does the right thing to do. The build system does
not need to manipulate the link order of head.o.
Link: https://lore.kernel.org/lkml/CAMj1kXH77Ja8bSsq2Qj8Ck9iSZKw=1F8Uy-uAWGVDm4-CG=EuA@mail.gmail.com/
Suggested-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
Reviewed-by: Nicolas Schier <nicolas@fjasle.eu>
Link: https://lore.kernel.org/r/20221012233500.156764-1-masahiroy@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
scripts/head-object-list.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/scripts/head-object-list.txt b/scripts/head-object-list.txt
index b16326a92c45..f226e45e3b7b 100644
--- a/scripts/head-object-list.txt
+++ b/scripts/head-object-list.txt
@@ -15,7 +15,6 @@ arch/alpha/kernel/head.o
arch/arc/kernel/head.o
arch/arm/kernel/head-nommu.o
arch/arm/kernel/head.o
-arch/arm64/kernel/head.o
arch/csky/kernel/head.o
arch/hexagon/kernel/head.o
arch/ia64/kernel/head.o
From ad51b5043bb39e0d715e8ad910fa5ac899ebc70b Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Fri, 30 Sep 2022 12:18:42 +0100
Subject: [PATCH 006/151] arm_pmu: acpi: factor out PMU<->CPU association
A subsequent patch will rework the ACPI probing of PMUs, and we'll need
to associate a CPU with a PMU in two separate paths.
Factor out the association logic into a helper function so that it can
be reused.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Pierre Gondois <pierre.gondois@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-and-tested-by: Pierre Gondois <pierre.gondois@arm.com>
Link: https://lore.kernel.org/r/20220930111844.1522365-2-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_pmu_acpi.c | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c
index 96ffadd654ff..a52a4aafd629 100644
--- a/drivers/perf/arm_pmu_acpi.c
+++ b/drivers/perf/arm_pmu_acpi.c
@@ -242,6 +242,22 @@ static bool pmu_irq_matches(struct arm_pmu *pmu, int irq)
return true;
}
+static void arm_pmu_acpi_associate_pmu_cpu(struct arm_pmu *pmu,
+ unsigned int cpu)
+{
+ int irq = per_cpu(pmu_irqs, cpu);
+
+ per_cpu(probed_pmus, cpu) = pmu;
+
+ if (pmu_irq_matches(pmu, irq)) {
+ struct pmu_hw_events __percpu *hw_events;
+ hw_events = pmu->hw_events;
+ per_cpu(hw_events->irq, cpu) = irq;
+ }
+
+ cpumask_set_cpu(cpu, &pmu->supported_cpus);
+}
+
/*
* This must run before the common arm_pmu hotplug logic, so that we can
* associate a CPU and its interrupt before the common code tries to manage the
@@ -254,27 +270,16 @@ static bool pmu_irq_matches(struct arm_pmu *pmu, int irq)
static int arm_pmu_acpi_cpu_starting(unsigned int cpu)
{
struct arm_pmu *pmu;
- struct pmu_hw_events __percpu *hw_events;
- int irq;
/* If we've already probed this CPU, we have nothing to do */
if (per_cpu(probed_pmus, cpu))
return 0;
- irq = per_cpu(pmu_irqs, cpu);
-
pmu = arm_pmu_acpi_find_alloc_pmu();
if (!pmu)
return -ENOMEM;
- per_cpu(probed_pmus, cpu) = pmu;
-
- if (pmu_irq_matches(pmu, irq)) {
- hw_events = pmu->hw_events;
- per_cpu(hw_events->irq, cpu) = irq;
- }
-
- cpumask_set_cpu(cpu, &pmu->supported_cpus);
+ arm_pmu_acpi_associate_pmu_cpu(pmu, cpu);
/*
* Ideally, we'd probe the PMU here when we find the first matching
From 6349a2470d07561bc25e34681985c6ff9c807dfb Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Fri, 30 Sep 2022 12:18:43 +0100
Subject: [PATCH 007/151] arm_pmu: factor out PMU matching
A subsequent patch will rework the ACPI probing of PMUs, and we'll need
to match a CPU with a known cpuid in two separate paths.
Factor out the matching logic into a helper function so that it can be
reused.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Pierre Gondois <pierre.gondois@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-and-tested-by: Pierre Gondois <pierre.gondois@arm.com>
Link: https://lore.kernel.org/r/20220930111844.1522365-3-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_pmu_acpi.c | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c
index a52a4aafd629..99abea3b2cc9 100644
--- a/drivers/perf/arm_pmu_acpi.c
+++ b/drivers/perf/arm_pmu_acpi.c
@@ -187,7 +187,7 @@ out_err:
return err;
}
-static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
+static struct arm_pmu *arm_pmu_acpi_find_pmu(void)
{
unsigned long cpuid = read_cpuid_id();
struct arm_pmu *pmu;
@@ -201,6 +201,17 @@ static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
return pmu;
}
+ return NULL;
+}
+
+static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
+{
+ struct arm_pmu *pmu;
+
+ pmu = arm_pmu_acpi_find_pmu();
+ if (pmu)
+ return pmu;
+
pmu = armpmu_alloc_atomic();
if (!pmu) {
pr_warn("Unable to allocate PMU for CPU%d\n",
@@ -208,7 +219,7 @@ static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
return NULL;
}
- pmu->acpi_cpuid = cpuid;
+ pmu->acpi_cpuid = read_cpuid_id();
return pmu;
}
From fe40ffdb7656d1f9c42dd402740765ff8b418b17 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Fri, 30 Sep 2022 12:18:44 +0100
Subject: [PATCH 008/151] arm_pmu: rework ACPI probing
The current ACPI PMU probing logic tries to associate PMUs with CPUs
when the CPU is first brought online, in order to handle late hotplug,
though PMUs are only registered during early boot, and so for late
hotplugged CPUs this can only associate the CPU with an existing PMU.
We tried to be clever and the have the arm_pmu_acpi_cpu_starting()
callback allocate a struct arm_pmu when no matching instance is found,
in order to avoid duplication of logic. However, as above this doesn't
do anything useful for late hotplugged CPUs, and this requires us to
allocate memory in an atomic context, which is especially problematic
for PREEMPT_RT, as reported by Valentin and Pierre.
This patch reworks the probing to detect PMUs for all online CPUs in the
arm_pmu_acpi_probe() function, which is more aligned with how DT probing
works. The arm_pmu_acpi_cpu_starting() callback only tries to associate
CPUs with an existing arm_pmu instance, avoiding the problem of
allocating in atomic context.
Note that as we didn't previously register PMUs for late-hotplugged
CPUs, this change doesn't result in a loss of existing functionality,
though we will now warn when we cannot associate a CPU with a PMU.
This change allows us to pull the hotplug callback registration into the
arm_pmu_acpi_probe() function, as we no longer need the callbacks to be
invoked shortly after probing the boot CPUs, and can register it without
invoking the calls.
For the moment the arm_pmu_acpi_init() initcall remains to register the
SPE PMU, though in future this should probably be moved elsewhere (e.g.
the arm64 ACPI init code), since this doesn't need to be tied to the
regular CPU PMU code.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reported-by: Valentin Schneider <valentin.schneider@arm.com>
Link: https://lore.kernel.org/r/20210810134127.1394269-2-valentin.schneider@arm.com/
Reported-by: Pierre Gondois <pierre.gondois@arm.com>
Link: https://lore.kernel.org/linux-arm-kernel/20220912155105.1443303-1-pierre.gondois@arm.com/
Cc: Pierre Gondois <pierre.gondois@arm.com>
Cc: Valentin Schneider <vschneid@redhat.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-and-tested-by: Pierre Gondois <pierre.gondois@arm.com>
Link: https://lore.kernel.org/r/20220930111844.1522365-4-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_pmu.c | 17 ++-----
drivers/perf/arm_pmu_acpi.c | 95 +++++++++++++++++++-----------------
include/linux/perf/arm_pmu.h | 1 -
3 files changed, 52 insertions(+), 61 deletions(-)
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 3f07df5a7e95..82a6d22e8ee2 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -861,16 +861,16 @@ static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
&cpu_pmu->node);
}
-static struct arm_pmu *__armpmu_alloc(gfp_t flags)
+struct arm_pmu *armpmu_alloc(void)
{
struct arm_pmu *pmu;
int cpu;
- pmu = kzalloc(sizeof(*pmu), flags);
+ pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
if (!pmu)
goto out;
- pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags);
+ pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, GFP_KERNEL);
if (!pmu->hw_events) {
pr_info("failed to allocate per-cpu PMU data.\n");
goto out_free_pmu;
@@ -916,17 +916,6 @@ out:
return NULL;
}
-struct arm_pmu *armpmu_alloc(void)
-{
- return __armpmu_alloc(GFP_KERNEL);
-}
-
-struct arm_pmu *armpmu_alloc_atomic(void)
-{
- return __armpmu_alloc(GFP_ATOMIC);
-}
-
-
void armpmu_free(struct arm_pmu *pmu)
{
free_percpu(pmu->hw_events);
diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c
index 99abea3b2cc9..a085e45b509e 100644
--- a/drivers/perf/arm_pmu_acpi.c
+++ b/drivers/perf/arm_pmu_acpi.c
@@ -13,6 +13,7 @@
#include <linux/percpu.h>
#include <linux/perf/arm_pmu.h>
+#include <asm/cpu.h>
#include <asm/cputype.h>
static DEFINE_PER_CPU(struct arm_pmu *, probed_pmus);
@@ -204,26 +205,6 @@ static struct arm_pmu *arm_pmu_acpi_find_pmu(void)
return NULL;
}
-static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
-{
- struct arm_pmu *pmu;
-
- pmu = arm_pmu_acpi_find_pmu();
- if (pmu)
- return pmu;
-
- pmu = armpmu_alloc_atomic();
- if (!pmu) {
- pr_warn("Unable to allocate PMU for CPU%d\n",
- smp_processor_id());
- return NULL;
- }
-
- pmu->acpi_cpuid = read_cpuid_id();
-
- return pmu;
-}
-
/*
* Check whether the new IRQ is compatible with those already associated with
* the PMU (e.g. we don't have mismatched PPIs).
@@ -286,26 +267,45 @@ static int arm_pmu_acpi_cpu_starting(unsigned int cpu)
if (per_cpu(probed_pmus, cpu))
return 0;
- pmu = arm_pmu_acpi_find_alloc_pmu();
- if (!pmu)
- return -ENOMEM;
+ pmu = arm_pmu_acpi_find_pmu();
+ if (!pmu) {
+ pr_warn_ratelimited("Unable to associate CPU%d with a PMU\n",
+ cpu);
+ return 0;
+ }
arm_pmu_acpi_associate_pmu_cpu(pmu, cpu);
-
- /*
- * Ideally, we'd probe the PMU here when we find the first matching
- * CPU. We can't do that for several reasons; see the comment in
- * arm_pmu_acpi_init().
- *
- * So for the time being, we're done.
- */
return 0;
}
+static void arm_pmu_acpi_probe_matching_cpus(struct arm_pmu *pmu,
+ unsigned long cpuid)
+{
+ int cpu;
+
+ for_each_online_cpu(cpu) {
+ unsigned long cpu_cpuid = per_cpu(cpu_data, cpu).reg_midr;
+
+ if (cpu_cpuid == cpuid)
+ arm_pmu_acpi_associate_pmu_cpu(pmu, cpu);
+ }
+}
+
int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
{
int pmu_idx = 0;
- int cpu, ret;
+ unsigned int cpu;
+ int ret;
+
+ ret = arm_pmu_acpi_parse_irqs();
+ if (ret)
+ return ret;
+
+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_ACPI_STARTING,
+ "perf/arm/pmu_acpi:starting",
+ arm_pmu_acpi_cpu_starting, NULL);
+ if (ret)
+ return ret;
/*
* Initialise and register the set of PMUs which we know about right
@@ -320,13 +320,26 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
* For the moment, as with the platform/DT case, we need at least one
* of a PMU's CPUs to be online at probe time.
*/
- for_each_possible_cpu(cpu) {
+ for_each_online_cpu(cpu) {
struct arm_pmu *pmu = per_cpu(probed_pmus, cpu);
+ unsigned long cpuid;
char *base_name;
- if (!pmu || pmu->name)
+ /* If we've already probed this CPU, we have nothing to do */
+ if (pmu)
continue;
+ pmu = armpmu_alloc();
+ if (!pmu) {
+ pr_warn("Unable to allocate PMU for CPU%d\n",
+ cpu);
+ }
+
+ cpuid = per_cpu(cpu_data, cpu).reg_midr;
+ pmu->acpi_cpuid = cpuid;
+
+ arm_pmu_acpi_probe_matching_cpus(pmu, cpuid);
+
ret = init_fn(pmu);
if (ret == -ENODEV) {
/* PMU not handled by this driver, or not present */
@@ -351,26 +364,16 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
}
}
- return 0;
+ return ret;
}
static int arm_pmu_acpi_init(void)
{
- int ret;
-
if (acpi_disabled)
return 0;
arm_spe_acpi_register_device();
- ret = arm_pmu_acpi_parse_irqs();
- if (ret)
- return ret;
-
- ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_ACPI_STARTING,
- "perf/arm/pmu_acpi:starting",
- arm_pmu_acpi_cpu_starting, NULL);
-
- return ret;
+ return 0;
}
subsys_initcall(arm_pmu_acpi_init)
diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h
index 0356cb6a215d..0c15c5b7f801 100644
--- a/include/linux/perf/arm_pmu.h
+++ b/include/linux/perf/arm_pmu.h
@@ -174,7 +174,6 @@ void kvm_host_pmu_init(struct arm_pmu *pmu);
/* Internal functions only for core arm_pmu code */
struct arm_pmu *armpmu_alloc(void);
-struct arm_pmu *armpmu_alloc_atomic(void);
void armpmu_free(struct arm_pmu *pmu);
int armpmu_register(struct arm_pmu *pmu);
int armpmu_request_irq(int irq, int cpu);
From e8e5104118473ffa23c013da42ae6a9df2867a07 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Wed, 19 Oct 2022 13:03:45 +0100
Subject: [PATCH 009/151] arm64/asm: Remove unused enable_da macro
We no longer use the enable_da macro, remove it to avoid having to think
about maintaining it.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221019120346.72289-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/assembler.h | 5 -----
1 file changed, 5 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index e5957a53be39..27e0c75f7818 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -51,11 +51,6 @@
msr daif, \flags
.endm
- /* IRQ/FIQ are the lowest priority flags, unconditionally unmask the rest. */
- .macro enable_da
- msr daifclr, #(8 | 4)
- .endm
-
/*
* Save/restore interrupts.
*/
From a8731264e5ce083eb761e5a6a8273db536743e9e Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Tue, 8 Nov 2022 09:37:25 +0000
Subject: [PATCH 010/151] arm_pmu: acpi: handle allocation failure
One of the failure paths in the arm_pmu ACPI code is missing an early
return, permitting a NULL pointer dereference upon a memory allocation
failure.
Add the missing return.
Fixes: fe40ffdb7656 ("arm_pmu: rework ACPI probing")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reported-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221108093725.1239563-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_pmu_acpi.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c
index a085e45b509e..90815ad762eb 100644
--- a/drivers/perf/arm_pmu_acpi.c
+++ b/drivers/perf/arm_pmu_acpi.c
@@ -333,6 +333,7 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
if (!pmu) {
pr_warn("Unable to allocate PMU for CPU%d\n",
cpu);
+ return -ENOMEM;
}
cpuid = per_cpu(cpu_data, cpu).reg_midr;
From 38e4b6605e5cde68ba388f3175e7b4962694674c Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Tue, 8 Nov 2022 09:14:06 +0530
Subject: [PATCH 011/151] arm64/mm: Drop ARM64_KERNEL_USES_PMD_MAPS
Currently ARM64_KERNEL_USES_PMD_MAPS is an unnecessary abstraction. Kernel
mapping at PMD (aka huge page aka block) level, is only applicable with 4K
base page, which makes it 2MB aligned, a necessary requirement for linear
mapping and physical memory start address. This can be easily achieved by
directly checking against base page size itself. This drops off the macro
ARM64_KERNE_USES_PMD_MAPS which is redundant.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: linux-arm-kernel@lists.infradead.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lore.kernel.org/r/20221108034406.2950071-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/kernel-pgtable.h | 11 +++--------
arch/arm64/mm/mmu.c | 2 +-
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 32d14f481f0c..fcd14197756f 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -18,11 +18,6 @@
* with 4K (section size = 2M) but not with 16K (section size = 32M) or
* 64K (section size = 512M).
*/
-#ifdef CONFIG_ARM64_4K_PAGES
-#define ARM64_KERNEL_USES_PMD_MAPS 1
-#else
-#define ARM64_KERNEL_USES_PMD_MAPS 0
-#endif
/*
* The idmap and swapper page tables need some space reserved in the kernel
@@ -34,7 +29,7 @@
* VA range, so pages required to map highest possible PA are reserved in all
* cases.
*/
-#if ARM64_KERNEL_USES_PMD_MAPS
+#ifdef CONFIG_ARM64_4K_PAGES
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
#else
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
@@ -96,7 +91,7 @@
#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE, 1)
/* Initial memory map size */
-#if ARM64_KERNEL_USES_PMD_MAPS
+#ifdef CONFIG_ARM64_4K_PAGES
#define SWAPPER_BLOCK_SHIFT PMD_SHIFT
#define SWAPPER_BLOCK_SIZE PMD_SIZE
#define SWAPPER_TABLE_SHIFT PUD_SHIFT
@@ -112,7 +107,7 @@
#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
-#if ARM64_KERNEL_USES_PMD_MAPS
+#ifdef CONFIG_ARM64_4K_PAGES
#define SWAPPER_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PMD_SECT_RDONLY)
#else
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9a7c38965154..d386033a074c 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1196,7 +1196,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
- if (!ARM64_KERNEL_USES_PMD_MAPS)
+ if (!IS_ENABLED(CONFIG_ARM64_4K_PAGES))
return vmemmap_populate_basepages(start, end, node, altmap);
do {
From 8a8112d888556b038687f3cfbb7c5a3f5fdb3116 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 7 Nov 2022 17:19:12 +0000
Subject: [PATCH 012/151] arm64/booting: Add missing colon to FA64 entry
The booting documentation for FEAT_SME_FA64 is missing a colon that is
normally present, add it.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221107171913.294565-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/booting.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst
index 8c324ad638de..62ddba739d99 100644
--- a/Documentation/arm64/booting.rst
+++ b/Documentation/arm64/booting.rst
@@ -348,7 +348,7 @@ Before jumping into the kernel, the following conditions must be met:
- HWFGWTR_EL2.nSMPRI_EL1 (bit 54) must be initialised to 0b01.
- For CPUs with the Scalable Matrix Extension FA64 feature (FEAT_SME_FA64)
+ For CPUs with the Scalable Matrix Extension FA64 feature (FEAT_SME_FA64):
- If EL3 is present:
From 59598b42eb52c734bd45d00f64a73129a4537e49 Mon Sep 17 00:00:00 2001
From: Mukesh Ojha <quic_mojha@quicinc.com>
Date: Sat, 29 Oct 2022 12:37:48 +0530
Subject: [PATCH 013/151] arm64: entry: Fix typo
Fix the following typo in entry-common.c
intrumentable => instrumentable
Signed-off-by: Mukesh Ojha <quic_mojha@quicinc.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/1667027268-1255-1-git-send-email-quic_mojha@quicinc.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/entry-common.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 27369fa1c032..9a553b185e63 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -30,7 +30,7 @@
/*
* Handle IRQ/context state management when entering from kernel mode.
* Before this function is called it is not safe to call regular kernel code,
- * intrumentable code, or any code which may trigger an exception.
+ * instrumentable code, or any code which may trigger an exception.
*
* This is intended to match the logic in irqentry_enter(), handling the kernel
* mode transitions only.
@@ -63,7 +63,7 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
/*
* Handle IRQ/context state management when exiting to kernel mode.
* After this function returns it is not safe to call regular kernel code,
- * intrumentable code, or any code which may trigger an exception.
+ * instrumentable code, or any code which may trigger an exception.
*
* This is intended to match the logic in irqentry_exit(), handling the kernel
* mode transitions only, and with preemption handled elsewhere.
@@ -97,7 +97,7 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
/*
* Handle IRQ/context state management when entering from user mode.
* Before this function is called it is not safe to call regular kernel code,
- * intrumentable code, or any code which may trigger an exception.
+ * instrumentable code, or any code which may trigger an exception.
*/
static __always_inline void __enter_from_user_mode(void)
{
@@ -116,7 +116,7 @@ static __always_inline void enter_from_user_mode(struct pt_regs *regs)
/*
* Handle IRQ/context state management when exiting to user mode.
* After this function returns it is not safe to call regular kernel code,
- * intrumentable code, or any code which may trigger an exception.
+ * instrumentable code, or any code which may trigger an exception.
*/
static __always_inline void __exit_to_user_mode(void)
{
@@ -152,7 +152,7 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
/*
* Handle IRQ/context state management when entering an NMI from user/kernel
* mode. Before this function is called it is not safe to call regular kernel
- * code, intrumentable code, or any code which may trigger an exception.
+ * code, instrumentable code, or any code which may trigger an exception.
*/
static void noinstr arm64_enter_nmi(struct pt_regs *regs)
{
@@ -170,7 +170,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs)
/*
* Handle IRQ/context state management when exiting an NMI from user/kernel
* mode. After this function returns it is not safe to call regular kernel
- * code, intrumentable code, or any code which may trigger an exception.
+ * code, instrumentable code, or any code which may trigger an exception.
*/
static void noinstr arm64_exit_nmi(struct pt_regs *regs)
{
@@ -192,7 +192,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs)
/*
* Handle IRQ/context state management when entering a debug exception from
* kernel mode. Before this function is called it is not safe to call regular
- * kernel code, intrumentable code, or any code which may trigger an exception.
+ * kernel code, instrumentable code, or any code which may trigger an exception.
*/
static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
{
@@ -207,7 +207,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
/*
* Handle IRQ/context state management when exiting a debug exception from
* kernel mode. After this function returns it is not safe to call regular
- * kernel code, intrumentable code, or any code which may trigger an exception.
+ * kernel code, instrumentable code, or any code which may trigger an exception.
*/
static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
{
From 01ab991fc0ee5019aecc4ca461311fc8aa75ece5 Mon Sep 17 00:00:00 2001
From: Ard Biesheuvel <ardb@kernel.org>
Date: Mon, 7 Nov 2022 18:24:00 +0100
Subject: [PATCH 014/151] arm64: Enable data independent timing (DIT) in the
kernel
The ARM architecture revision v8.4 introduces a data independent timing
control (DIT) which can be set at any exception level, and instructs the
CPU to avoid optimizations that may result in a correlation between the
execution time of certain instructions and the value of the data they
operate on.
The DIT bit is part of PSTATE, and is therefore context switched as
usual, given that it becomes part of the saved program state (SPSR) when
taking an exception. We have also defined a hwcap for DIT, and so user
space can discover already whether or nor DIT is available. This means
that, as far as user space is concerned, DIT is wired up and fully
functional.
In the kernel, however, we never bothered with DIT: we disable at it
boot (i.e., INIT_PSTATE_EL1 has DIT cleared) and ignore the fact that we
might run with DIT enabled if user space happened to set it.
Currently, we have no idea whether or not running privileged code with
DIT disabled on a CPU that implements support for it may result in a
side channel that exposes privileged data to unprivileged user space
processes, so let's be cautious and just enable DIT while running in the
kernel if supported by all CPUs.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Eric Biggers <ebiggers@kernel.org>
Cc: Jason A. Donenfeld <Jason@zx2c4.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Adam Langley <agl@google.com>
Link: https://lore.kernel.org/all/YwgCrqutxmX0W72r@gmail.com/
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221107172400.1851434-1-ardb@kernel.org
[will: Removed cpu_has_dit() as per Mark's suggestion on the list]
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 12 ++++++++----
arch/arm64/kernel/cpufeature.c | 17 +++++++++++++++++
arch/arm64/kernel/entry.S | 3 +++
arch/arm64/kernel/suspend.c | 2 ++
arch/arm64/tools/cpucaps | 1 +
5 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7d301700d1a9..1f3f52ce407f 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -90,20 +90,24 @@
*/
#define pstate_field(op1, op2) ((op1) << Op1_shift | (op2) << Op2_shift)
#define PSTATE_Imm_shift CRm_shift
+#define SET_PSTATE(x, r) __emit_inst(0xd500401f | PSTATE_ ## r | ((!!x) << PSTATE_Imm_shift))
#define PSTATE_PAN pstate_field(0, 4)
#define PSTATE_UAO pstate_field(0, 3)
#define PSTATE_SSBS pstate_field(3, 1)
+#define PSTATE_DIT pstate_field(3, 2)
#define PSTATE_TCO pstate_field(3, 4)
-#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f | PSTATE_PAN | ((!!x) << PSTATE_Imm_shift))
-#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift))
-#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift))
-#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f | PSTATE_TCO | ((!!x) << PSTATE_Imm_shift))
+#define SET_PSTATE_PAN(x) SET_PSTATE((x), PAN)
+#define SET_PSTATE_UAO(x) SET_PSTATE((x), UAO)
+#define SET_PSTATE_SSBS(x) SET_PSTATE((x), SSBS)
+#define SET_PSTATE_DIT(x) SET_PSTATE((x), DIT)
+#define SET_PSTATE_TCO(x) SET_PSTATE((x), TCO)
#define set_pstate_pan(x) asm volatile(SET_PSTATE_PAN(x))
#define set_pstate_uao(x) asm volatile(SET_PSTATE_UAO(x))
#define set_pstate_ssbs(x) asm volatile(SET_PSTATE_SSBS(x))
+#define set_pstate_dit(x) asm volatile(SET_PSTATE_DIT(x))
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b3f37e2209ad..3086304652fd 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2101,6 +2101,11 @@ static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused)
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP);
}
+static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
+{
+ set_pstate_dit(1);
+}
+
/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
@@ -2664,6 +2669,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_cpuid_feature,
.cpu_enable = cpu_trap_el0_impdef,
},
+ {
+ .desc = "Data independent timing control (DIT)",
+ .capability = ARM64_HAS_DIT,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .sys_reg = SYS_ID_AA64PFR0_EL1,
+ .sign = FTR_UNSIGNED,
+ .field_pos = ID_AA64PFR0_EL1_DIT_SHIFT,
+ .field_width = 4,
+ .min_field_value = ID_AA64PFR0_EL1_DIT_IMP,
+ .matches = has_cpuid_feature,
+ .cpu_enable = cpu_enable_dit,
+ },
{},
};
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index e28137d64b76..11cb99c4d298 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -197,6 +197,9 @@ alternative_cb_end
.endm
.macro kernel_entry, el, regsize = 64
+ .if \el == 0
+ alternative_insn nop, SET_PSTATE_DIT(1), ARM64_HAS_DIT
+ .endif
.if \regsize == 32
mov w0, w0 // zero upper 32 bits of x0
.endif
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 8b02d310838f..e7163f31f716 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -60,6 +60,8 @@ void notrace __cpu_suspend_exit(void)
* PSTATE was not saved over suspend/resume, re-enable any detected
* features that might not have been set correctly.
*/
+ if (cpus_have_const_cap(ARM64_HAS_DIT))
+ set_pstate_dit(1);
__uaccess_enable_hw_pan();
/*
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index f1c0347ec31a..a86ee376920a 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -20,6 +20,7 @@ HAS_CNP
HAS_CRC32
HAS_DCPODP
HAS_DCPOP
+HAS_DIT
HAS_E0PD
HAS_ECV
HAS_EPAN
From 3a38ef2b3cb6b63c105247b5ea4a9cf600e673f0 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 15:45:52 +0100
Subject: [PATCH 015/151] kselftest/arm64: Check that all children are
producing output in fp-stress
Currently we don't have an explicit check that when it's been a second
since we have seen output produced from the test programs starting up that
means all of them are running and we should start both sending signals and
timing out. This is not reliable, especially on very heavily loaded systems
where the test programs might take longer than a second to run.
We do skip sending signals to children that have not produced output yet
so we won't cause them to exit unexpectedly by sending a signal but this
can create confusion when interpreting output, for example appearing to
show the tests running for less time than expected or appearing to show
missed signal deliveries. Avoid issues by explicitly checking that we have
seen output from all the child processes before we start sending signals
or counting test run time.
This is especially likely on virtual platforms with large numbers of vector
lengths supported since the platforms are slow and there will be a lot of
tasks per CPU.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221017144553.773176-2-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 23 ++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index 4e62a9199f97..35dc07648d52 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -403,6 +403,8 @@ int main(int argc, char **argv)
int timeout = 10;
int cpus, tests, i, j, c;
int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
+ bool all_children_started = false;
+ int seen_children;
int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
struct epoll_event ev;
struct sigaction sa;
@@ -526,6 +528,27 @@ int main(int argc, char **argv)
/* Otherwise epoll_wait() timed out */
+ /*
+ * If the child processes have not produced output they
+ * aren't actually running the tests yet .
+ */
+ if (!all_children_started) {
+ seen_children = 0;
+
+ for (i = 0; i < num_children; i++)
+ if (children[i].output_seen ||
+ children[i].exited)
+ seen_children++;
+
+ if (seen_children != num_children) {
+ ksft_print_msg("Waiting for %d children\n",
+ num_children - seen_children);
+ continue;
+ }
+
+ all_children_started = true;
+ }
+
for (i = 0; i < num_children; i++)
child_tickle(&children[i]);
From 3e02f57bcc6a34ab992c4027b6ef3f2916b20924 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 15:45:53 +0100
Subject: [PATCH 016/151] kselftest/arm64: Provide progress messages when
signalling children
Especially when the test is configured to run for a longer time it can be
reassuring to users to see that the supervising program is running OK so
provide a message every second when the output timer expires.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221017144553.773176-3-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index 35dc07648d52..ccbfcf847d96 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -549,6 +549,9 @@ int main(int argc, char **argv)
all_children_started = true;
}
+ ksft_print_msg("Sending signals, timeout remaining: %d\n",
+ timeout);
+
for (i = 0; i < num_children; i++)
child_tickle(&children[i]);
From 2004734fb3fe088873ad4c5276cc92ee956e640d Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Thu, 27 Oct 2022 12:03:24 +0100
Subject: [PATCH 017/151] kselftest/arm64: Remove validation of extra_context
from TODO
When fixing up support for extra_context in the signal handling tests I
didn't notice that there is a TODO file in the directory which lists this
as a thing to be done. Since it's been done remove it from the list.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221027110324.33802-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/signal/testcases/TODO | 1 -
1 file changed, 1 deletion(-)
diff --git a/tools/testing/selftests/arm64/signal/testcases/TODO b/tools/testing/selftests/arm64/signal/testcases/TODO
index 110ff9fd195d..1f7fba8194fe 100644
--- a/tools/testing/selftests/arm64/signal/testcases/TODO
+++ b/tools/testing/selftests/arm64/signal/testcases/TODO
@@ -1,2 +1 @@
- Validate that register contents are saved and restored as expected.
-- Support and validate extra_context.
From 9b283888a6d5d0e413fd9bf097fd74c27a9b9948 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Wed, 2 Nov 2022 14:05:43 +0000
Subject: [PATCH 018/151] kselftest/arm64: Print ASCII version of unknown
signal frame magic values
The signal magic values are supposed to be allocated as somewhat meaningful
ASCII so if we encounter a bad magic value print the any alphanumeric
characters we find in it as well as the hex value to aid debuggability.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221102140543.98193-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
.../arm64/signal/testcases/testcases.c | 21 +++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.c b/tools/testing/selftests/arm64/signal/testcases/testcases.c
index e1c625b20ac4..d2eda7b5de26 100644
--- a/tools/testing/selftests/arm64/signal/testcases/testcases.c
+++ b/tools/testing/selftests/arm64/signal/testcases/testcases.c
@@ -1,5 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2019 ARM Limited */
+
+#include <ctype.h>
+#include <string.h>
+
#include "testcases.h"
struct _aarch64_ctx *get_header(struct _aarch64_ctx *head, uint32_t magic,
@@ -109,7 +113,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
bool terminated = false;
size_t offs = 0;
int flags = 0;
- int new_flags;
+ int new_flags, i;
struct extra_context *extra = NULL;
struct sve_context *sve = NULL;
struct za_context *za = NULL;
@@ -117,6 +121,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
(struct _aarch64_ctx *)uc->uc_mcontext.__reserved;
void *extra_data = NULL;
size_t extra_sz = 0;
+ char magic[4];
if (!err)
return false;
@@ -194,11 +199,19 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
/*
* A still unknown Magic: potentially freshly added
* to the Kernel code and still unknown to the
- * tests.
+ * tests. Magic numbers are supposed to be allocated
+ * as somewhat meaningful ASCII strings so try to
+ * print as such as well as the raw number.
*/
+ memcpy(magic, &head->magic, sizeof(magic));
+ for (i = 0; i < sizeof(magic); i++)
+ if (!isalnum(magic[i]))
+ magic[i] = '?';
+
fprintf(stdout,
- "SKIP Unknown MAGIC: 0x%X - Is KSFT arm64/signal up to date ?\n",
- head->magic);
+ "SKIP Unknown MAGIC: 0x%X (%c%c%c%c) - Is KSFT arm64/signal up to date ?\n",
+ head->magic,
+ magic[3], magic[2], magic[1], magic[0]);
break;
}
From a0fa0b63131b0d8f4eff22d5b66e796ecb064dfe Mon Sep 17 00:00:00 2001
From: Kang Minchul <tegongkang@gmail.com>
Date: Sat, 5 Nov 2022 16:31:43 +0900
Subject: [PATCH 019/151] kselftest/arm64: fix array_size.cocci warning
Use ARRAY_SIZE to fix the following coccicheck warnings:
tools/testing/selftests/arm64/mte/check_buffer_fill.c:341:20-21:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_buffer_fill.c:35:20-21:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_buffer_fill.c:168:20-21:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_buffer_fill.c:72:20-21:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_buffer_fill.c:369:25-26:
WARNING: Use ARRAY_SIZE
Signed-off-by: Kang Minchul <tegongkang@gmail.com>
Link: https://lore.kernel.org/r/20221105073143.78521-1-tegongkang@gmail.com
Signed-off-by: Will Deacon <will@kernel.org>
---
.../testing/selftests/arm64/mte/check_buffer_fill.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/testing/selftests/arm64/mte/check_buffer_fill.c b/tools/testing/selftests/arm64/mte/check_buffer_fill.c
index 75fc482d63b6..1dbbbd47dd50 100644
--- a/tools/testing/selftests/arm64/mte/check_buffer_fill.c
+++ b/tools/testing/selftests/arm64/mte/check_buffer_fill.c
@@ -32,7 +32,7 @@ static int check_buffer_by_byte(int mem_type, int mode)
bool err;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
- item = sizeof(sizes)/sizeof(int);
+ item = ARRAY_SIZE(sizes);
for (i = 0; i < item; i++) {
ptr = (char *)mte_allocate_memory(sizes[i], mem_type, 0, true);
@@ -69,7 +69,7 @@ static int check_buffer_underflow_by_byte(int mem_type, int mode,
char *und_ptr = NULL;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
- item = sizeof(sizes)/sizeof(int);
+ item = ARRAY_SIZE(sizes);
for (i = 0; i < item; i++) {
ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
underflow_range, 0);
@@ -165,7 +165,7 @@ static int check_buffer_overflow_by_byte(int mem_type, int mode,
char *over_ptr = NULL;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
- item = sizeof(sizes)/sizeof(int);
+ item = ARRAY_SIZE(sizes);
for (i = 0; i < item; i++) {
ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
0, overflow_range);
@@ -338,7 +338,7 @@ static int check_buffer_by_block(int mem_type, int mode)
int i, item, result = KSFT_PASS;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
- item = sizeof(sizes)/sizeof(int);
+ item = ARRAY_SIZE(sizes);
cur_mte_cxt.fault_valid = false;
for (i = 0; i < item; i++) {
result = check_buffer_by_block_iterate(mem_type, mode, sizes[i]);
@@ -366,7 +366,7 @@ static int check_memory_initial_tags(int mem_type, int mode, int mapping)
{
char *ptr;
int run, fd;
- int total = sizeof(sizes)/sizeof(int);
+ int total = ARRAY_SIZE(sizes);
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
for (run = 0; run < total; run++) {
@@ -404,7 +404,7 @@ int main(int argc, char *argv[])
{
int err;
size_t page_size = getpagesize();
- int item = sizeof(sizes)/sizeof(int);
+ int item = ARRAY_SIZE(sizes);
sizes[item - 3] = page_size - 1;
sizes[item - 2] = page_size;
From aaeca98456431a8d9382ecf48ac4843e252c07b3 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 7 Nov 2022 17:07:47 +0000
Subject: [PATCH 020/151] arm64/fpsimd: Make kernel_neon_ API _GPL
Currently for reasons lost in the mists of time the kernel_neon_ APIs are
EXPORT_SYMBOL() but the general policy for floating point usage is that it
should be GPL only given the non-standard runtime environment that holds
while it is in use and PCS impacts when code is compiled for FP usage.
Given the limited existing deployment of non-GPL modules for arm64 and the
fact that other architectures like x86 already make their equivalent
functions GPL only this is not expected to be disruptive to existing users.
Suggested-by: Christoph Hellwig <hch@infradead.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221107170747.276910-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 23834d96d1e7..1fe7d65096ac 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1838,7 +1838,7 @@ void kernel_neon_begin(void)
/* Invalidate any task state remaining in the fpsimd regs: */
fpsimd_flush_cpu_state();
}
-EXPORT_SYMBOL(kernel_neon_begin);
+EXPORT_SYMBOL_GPL(kernel_neon_begin);
/*
* kernel_neon_end(): give the CPU FPSIMD registers back to the current task
@@ -1856,7 +1856,7 @@ void kernel_neon_end(void)
put_cpu_fpsimd_context();
}
-EXPORT_SYMBOL(kernel_neon_end);
+EXPORT_SYMBOL_GPL(kernel_neon_end);
#ifdef CONFIG_EFI
From 95aa6860d60881e8f08956e02199f7d65c9cbf7b Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:15 +0100
Subject: [PATCH 021/151] arm64/hwcap: Add support for FEAT_CSSC
FEAT_CSSC adds a number of new instructions usable to optimise common short
sequences of instructions, add a hwcap indicating that the feature is
available and can be used by userspace.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-2-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/elf_hwcaps.rst | 3 +++
arch/arm64/include/asm/hwcap.h | 1 +
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 2 ++
arch/arm64/kernel/cpuinfo.c | 1 +
arch/arm64/tools/sysreg | 7 ++++++-
6 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index bb34287c8e01..58197e9ccb6d 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -275,6 +275,9 @@ HWCAP2_EBF16
HWCAP2_SVE_EBF16
Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0010.
+HWCAP2_CSSC
+ Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0001.
+
4. Unused AT_HWCAP bits
-----------------------
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index 298b386d3ebe..a0e080df9a62 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -120,6 +120,7 @@
#define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT)
#define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16)
#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
+#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
/*
* This yields a mask that user programs can use to figure out what
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 9b245da6f507..a43dddd94b4a 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -93,5 +93,6 @@
#define HWCAP2_WFXT (1UL << 31)
#define HWCAP2_EBF16 (1UL << 32)
#define HWCAP2_SVE_EBF16 (1UL << 33)
+#define HWCAP2_CSSC (1UL << 34)
#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 3086304652fd..23d1d6b01c1f 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -212,6 +212,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
};
static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
@@ -2815,6 +2816,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
#endif /* CONFIG_ARM64_MTE */
HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
+ HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_CSSC_IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC),
HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
#ifdef CONFIG_ARM64_SME
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 28d4f442b0bc..3160550c0cc9 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -116,6 +116,7 @@ static const char *const hwcap_str[] = {
[KERNEL_HWCAP_WFXT] = "wfxt",
[KERNEL_HWCAP_EBF16] = "ebf16",
[KERNEL_HWCAP_SVE_EBF16] = "sveebf16",
+ [KERNEL_HWCAP_CSSC] = "cssc",
};
#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 384757a7eda9..629d119151bf 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -484,7 +484,12 @@ EndEnum
EndSysreg
Sysreg ID_AA64ISAR2_EL1 3 0 0 6 2
-Res0 63:28
+Res0 63:56
+Enum 55:52 CSSC
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Res0 51:28
Enum 27:24 PAC_frac
0b0000 NI
0b0001 IMP
From b0ab73a5479fdf6b42babaccf22b4fa88f5a20a6 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:16 +0100
Subject: [PATCH 022/151] kselftest/arm64: Add FEAT_CSSC to the hwcap selftest
Add FEAT_CSSC to the set of features checked by the hwcap selftest.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-3-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/abi/hwcap.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c
index 9f1a7b5c6193..c7a6b327a7d0 100644
--- a/tools/testing/selftests/arm64/abi/hwcap.c
+++ b/tools/testing/selftests/arm64/abi/hwcap.c
@@ -33,6 +33,12 @@
*/
typedef void (*sigill_fn)(void);
+static void cssc_sigill(void)
+{
+ /* CNT x0, x0 */
+ asm volatile(".inst 0xdac01c00" : : : "x0");
+}
+
static void rng_sigill(void)
{
asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0");
@@ -118,6 +124,13 @@ static const struct hwcap_data {
sigill_fn sigill_fn;
bool sigill_reliable;
} hwcaps[] = {
+ {
+ .name = "CSSC",
+ .at_hwcap = AT_HWCAP2,
+ .hwcap_bit = HWCAP2_CSSC,
+ .cpuinfo = "cssc",
+ .sigill_fn = cssc_sigill,
+ },
{
.name = "RNG",
.at_hwcap = AT_HWCAP2,
From 939e4649d4fd54b622d08cd57100828df7a82074 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:17 +0100
Subject: [PATCH 023/151] arm64/hwcap: Add support for FEAT_RPRFM
FEAT_RPRFM adds a new range prefetch hint within the existing PRFM space
for range prefetch hinting. Add a new hwcap to allow userspace to discover
support for the new instruction.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-4-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/elf_hwcaps.rst | 3 +++
arch/arm64/include/asm/hwcap.h | 1 +
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 2 ++
arch/arm64/kernel/cpuinfo.c | 1 +
arch/arm64/tools/sysreg | 6 +++++-
6 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index 58197e9ccb6d..a82b2cdff680 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -278,6 +278,9 @@ HWCAP2_SVE_EBF16
HWCAP2_CSSC
Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0001.
+HWCAP2_RPRFM
+ Functionality implied by ID_AA64ISAR2_EL1.RPRFM == 0b0001.
+
4. Unused AT_HWCAP bits
-----------------------
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index a0e080df9a62..605ec55cee70 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -121,6 +121,7 @@
#define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16)
#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
+#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM)
/*
* This yields a mask that user programs can use to figure out what
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index a43dddd94b4a..063cc6ea560f 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -94,5 +94,6 @@
#define HWCAP2_EBF16 (1UL << 32)
#define HWCAP2_SVE_EBF16 (1UL << 33)
#define HWCAP2_CSSC (1UL << 34)
+#define HWCAP2_RPRFM (1UL << 35)
#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 23d1d6b01c1f..6e0837212e97 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -213,6 +213,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
@@ -2817,6 +2818,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_CSSC_IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC),
+ HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_RPRFM_IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM),
HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
#ifdef CONFIG_ARM64_SME
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 3160550c0cc9..85108832d86e 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -117,6 +117,7 @@ static const char *const hwcap_str[] = {
[KERNEL_HWCAP_EBF16] = "ebf16",
[KERNEL_HWCAP_SVE_EBF16] = "sveebf16",
[KERNEL_HWCAP_CSSC] = "cssc",
+ [KERNEL_HWCAP_RPRFM] = "rprfm",
};
#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 629d119151bf..a2b2e4c1c3f2 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -489,7 +489,11 @@ Enum 55:52 CSSC
0b0000 NI
0b0001 IMP
EndEnum
-Res0 51:28
+Enum 51:48 RPRFM
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Res0 47:28
Enum 27:24 PAC_frac
0b0000 NI
0b0001 IMP
From 989d37fc3d976f6bfa3f1da484f8f22c57c21c0c Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:18 +0100
Subject: [PATCH 024/151] kselftest/arm64: Add FEAT_RPRFM to the hwcap test
Since the newly added instruction is in the HINT space we can't reasonably
test for it actually being present.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-5-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/abi/hwcap.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c
index c7a6b327a7d0..30f87dfd634e 100644
--- a/tools/testing/selftests/arm64/abi/hwcap.c
+++ b/tools/testing/selftests/arm64/abi/hwcap.c
@@ -138,6 +138,12 @@ static const struct hwcap_data {
.cpuinfo = "rng",
.sigill_fn = rng_sigill,
},
+ {
+ .name = "RPRFM",
+ .at_hwcap = AT_HWCAP2,
+ .hwcap_bit = HWCAP2_RPRFM,
+ .cpuinfo = "rprfm",
+ },
{
.name = "SME",
.at_hwcap = AT_HWCAP2,
From d12aada8dfb030f7df8ec0bb0ce4bb01a2ab3944 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:19 +0100
Subject: [PATCH 025/151] arm64/hwcap: Add support for SVE 2.1
FEAT_SVE2p1 introduces a number of new SVE instructions. Since there is no
new architectural state added kernel support is simply a new hwcap which
lets userspace know that the feature is supported.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-6-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/elf_hwcaps.rst | 3 +++
Documentation/arm64/sve.rst | 1 +
arch/arm64/include/asm/hwcap.h | 1 +
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 1 +
arch/arm64/kernel/cpuinfo.c | 1 +
arch/arm64/tools/sysreg | 1 +
7 files changed, 9 insertions(+)
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index a82b2cdff680..6fed84f935df 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -281,6 +281,9 @@ HWCAP2_CSSC
HWCAP2_RPRFM
Functionality implied by ID_AA64ISAR2_EL1.RPRFM == 0b0001.
+HWCAP2_SVE2P1
+ Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0010.
+
4. Unused AT_HWCAP bits
-----------------------
diff --git a/Documentation/arm64/sve.rst b/Documentation/arm64/sve.rst
index f338ee2df46d..c7a356bf4e8f 100644
--- a/Documentation/arm64/sve.rst
+++ b/Documentation/arm64/sve.rst
@@ -52,6 +52,7 @@ model features for SVE is included in Appendix A.
HWCAP2_SVEBITPERM
HWCAP2_SVESHA3
HWCAP2_SVESM4
+ HWCAP2_SVE2P1
This list may be extended over time as the SVE architecture evolves.
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index 605ec55cee70..06dd12c514e6 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -122,6 +122,7 @@
#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM)
+#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1)
/*
* This yields a mask that user programs can use to figure out what
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 063cc6ea560f..b713d30544f1 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -95,5 +95,6 @@
#define HWCAP2_SVE_EBF16 (1UL << 33)
#define HWCAP2_CSSC (1UL << 34)
#define HWCAP2_RPRFM (1UL << 35)
+#define HWCAP2_SVE2P1 (1UL << 36)
#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 6e0837212e97..3e3af6b8cc85 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2791,6 +2791,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT),
#ifdef CONFIG_ARM64_SVE
HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_EL1_SVE_IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
+ HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 85108832d86e..379695262b77 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -118,6 +118,7 @@ static const char *const hwcap_str[] = {
[KERNEL_HWCAP_SVE_EBF16] = "sveebf16",
[KERNEL_HWCAP_CSSC] = "cssc",
[KERNEL_HWCAP_RPRFM] = "rprfm",
+ [KERNEL_HWCAP_SVE2P1] = "sve2p1",
};
#ifdef CONFIG_COMPAT
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index a2b2e4c1c3f2..b2782b8faa01 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -210,6 +210,7 @@ EndEnum
Enum 3:0 SVEver
0b0000 IMP
0b0001 SVE2
+ 0b0010 SVE2p1
EndEnum
EndSysreg
From c5195b027d29bbaae0c9668704ab996773788d23 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 17 Oct 2022 16:25:20 +0100
Subject: [PATCH 026/151] kselftest/arm64: Add SVE 2.1 to hwcap test
Add coverage for FEAT_SVE2p1.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221017152520.1039165-7-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/abi/hwcap.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c
index 30f87dfd634e..9f255bc5f31c 100644
--- a/tools/testing/selftests/arm64/abi/hwcap.c
+++ b/tools/testing/selftests/arm64/abi/hwcap.c
@@ -62,6 +62,12 @@ static void sve2_sigill(void)
asm volatile(".inst 0x4408A000" : : : "z0");
}
+static void sve2p1_sigill(void)
+{
+ /* BFADD Z0.H, Z0.H, Z0.H */
+ asm volatile(".inst 0x65000000" : : : "z0");
+}
+
static void sveaes_sigill(void)
{
/* AESD z0.b, z0.b, z0.b */
@@ -167,6 +173,13 @@ static const struct hwcap_data {
.cpuinfo = "sve2",
.sigill_fn = sve2_sigill,
},
+ {
+ .name = "SVE 2.1",
+ .at_hwcap = AT_HWCAP2,
+ .hwcap_bit = HWCAP2_SVE2P1,
+ .cpuinfo = "sve2p1",
+ .sigill_fn = sve2p1_sigill,
+ },
{
.name = "SVE AES",
.at_hwcap = AT_HWCAP2,
From 68c76ad4a9571a2b603665c85cf8229bcf04982a Mon Sep 17 00:00:00 2001
From: Ard Biesheuvel <ardb@kernel.org>
Date: Thu, 27 Oct 2022 17:59:06 +0200
Subject: [PATCH 027/151] arm64: unwind: add asynchronous unwind tables to
kernel and modules
Enable asynchronous unwind table generation for both the core kernel as
well as modules, and emit the resulting .eh_frame sections as init code
so we can use the unwind directives for code patching at boot or module
load time.
This will be used by dynamic shadow call stack support, which will rely
on code patching rather than compiler codegen to emit the shadow call
stack push and pop instructions.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Nick Desaulniers <ndesaulniers@google.com>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Tested-by: Sami Tolvanen <samitolvanen@google.com>
Link: https://lore.kernel.org/r/20221027155908.1940624-2-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 3 +++
arch/arm64/Makefile | 5 +++++
arch/arm64/include/asm/module.lds.h | 8 ++++++++
arch/arm64/kernel/pi/Makefile | 1 +
arch/arm64/kernel/vmlinux.lds.S | 13 +++++++++++++
arch/arm64/kvm/hyp/nvhe/Makefile | 1 +
drivers/firmware/efi/libstub/Makefile | 1 +
include/asm-generic/vmlinux.lds.h | 9 +++++++--
scripts/module.lds.S | 6 ++++++
9 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 505c8a1ccbe0..7e3a9cf2193d 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -370,6 +370,9 @@ config KASAN_SHADOW_OFFSET
default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
default 0xffffffffffffffff
+config UNWIND_TABLES
+ bool
+
source "arch/arm64/Kconfig.platforms"
menu "Kernel Features"
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 5e56d26a2239..7868a176993f 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -45,8 +45,13 @@ KBUILD_CFLAGS += $(call cc-option,-mabi=lp64)
KBUILD_AFLAGS += $(call cc-option,-mabi=lp64)
# Avoid generating .eh_frame* sections.
+ifneq ($(CONFIG_UNWIND_TABLES),y)
KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
KBUILD_AFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
+else
+KBUILD_CFLAGS += -fasynchronous-unwind-tables
+KBUILD_AFLAGS += -fasynchronous-unwind-tables
+endif
ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y)
prepare: stack_protector_prepare
diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h
index 094701ec5500..dbba4b7559aa 100644
--- a/arch/arm64/include/asm/module.lds.h
+++ b/arch/arm64/include/asm/module.lds.h
@@ -17,4 +17,12 @@ SECTIONS {
*/
.text.hot : { *(.text.hot) }
#endif
+
+#ifdef CONFIG_UNWIND_TABLES
+ /*
+ * Currently, we only use unwind info at module load time, so we can
+ * put it into the .init allocation.
+ */
+ .init.eh_frame : { *(.eh_frame) }
+#endif
}
diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile
index 839291430cb3..4c0ea3cd4ea4 100644
--- a/arch/arm64/kernel/pi/Makefile
+++ b/arch/arm64/kernel/pi/Makefile
@@ -7,6 +7,7 @@ KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \
-I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \
-include $(srctree)/include/linux/hidden.h \
-D__DISABLE_EXPORTS -ffreestanding -D__NO_FORTIFY \
+ -fno-asynchronous-unwind-tables -fno-unwind-tables \
$(call cc-option,-fno-addrsig)
# remove SCS flags from all objects in this directory
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 45131e354e27..4c13dafc98b8 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -121,6 +121,17 @@ jiffies = jiffies_64;
#define TRAMP_TEXT
#endif
+#ifdef CONFIG_UNWIND_TABLES
+#define UNWIND_DATA_SECTIONS \
+ .eh_frame : { \
+ __eh_frame_start = .; \
+ *(.eh_frame) \
+ __eh_frame_end = .; \
+ }
+#else
+#define UNWIND_DATA_SECTIONS
+#endif
+
/*
* The size of the PE/COFF section that covers the kernel image, which
* runs from _stext to _edata, must be a round multiple of the PE/COFF
@@ -231,6 +242,8 @@ SECTIONS
__alt_instructions_end = .;
}
+ UNWIND_DATA_SECTIONS
+
. = ALIGN(SEGMENT_ALIGN);
__inittext_end = .;
__initdata_begin = .;
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index be0a2bc3e20d..530347cdebe3 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -96,6 +96,7 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI)
# when profile optimization is applied. gen-hyprel does not support SHT_REL and
# causes a build failure. Remove profile optimization flags.
KBUILD_CFLAGS := $(filter-out -fprofile-sample-use=% -fprofile-use=%, $(KBUILD_CFLAGS))
+KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
# KVM nVHE code is run at a different exception code with a different map, so
# compiler instrumentation that inserts callbacks or checks into the code may
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index b1601aad7e1a..1016f0b5311d 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -20,6 +20,7 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \
# disable the stackleak plugin
cflags-$(CONFIG_ARM64) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
-fpie $(DISABLE_STACKLEAK_PLUGIN) \
+ -fno-unwind-tables -fno-asynchronous-unwind-tables \
$(call cc-option,-mbranch-protection=none)
cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
-fno-builtin -fpic \
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index d06ada2341cb..0cca179e5106 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -1027,14 +1027,19 @@
* keep any .init_array.* sections.
* https://bugs.llvm.org/show_bug.cgi?id=46478
*/
+#ifdef CONFIG_UNWIND_TABLES
+#define DISCARD_EH_FRAME
+#else
+#define DISCARD_EH_FRAME *(.eh_frame)
+#endif
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
# ifdef CONFIG_CONSTRUCTORS
# define SANITIZER_DISCARDS \
- *(.eh_frame)
+ DISCARD_EH_FRAME
# else
# define SANITIZER_DISCARDS \
*(.init_array) *(.init_array.*) \
- *(.eh_frame)
+ DISCARD_EH_FRAME
# endif
#else
# define SANITIZER_DISCARDS
diff --git a/scripts/module.lds.S b/scripts/module.lds.S
index da4bddd26171..bf5bcf2836d8 100644
--- a/scripts/module.lds.S
+++ b/scripts/module.lds.S
@@ -3,6 +3,12 @@
* Archs are free to supply their own linker scripts. ld will
* combine them automatically.
*/
+#ifdef CONFIG_UNWIND_TABLES
+#define DISCARD_EH_FRAME
+#else
+#define DISCARD_EH_FRAME *(.eh_frame)
+#endif
+
SECTIONS {
/DISCARD/ : {
*(.discard)
From 9beccca0984022a844850e32f0d7dd80d4a225de Mon Sep 17 00:00:00 2001
From: Ard Biesheuvel <ardb@kernel.org>
Date: Thu, 27 Oct 2022 17:59:07 +0200
Subject: [PATCH 028/151] scs: add support for dynamic shadow call stacks
In order to allow arches to use code patching to conditionally emit the
shadow stack pushes and pops, rather than always taking the performance
hit even on CPUs that implement alternatives such as stack pointer
authentication on arm64, add a Kconfig symbol that can be set by the
arch to omit the SCS codegen itself, without otherwise affecting how
support code for SCS and compiler options (for register reservation, for
instance) are emitted.
Also, add a static key and some plumbing to omit the allocation of
shadow call stack for dynamic SCS configurations if SCS is disabled at
runtime.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Nick Desaulniers <ndesaulniers@google.com>
Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Tested-by: Sami Tolvanen <samitolvanen@google.com>
Link: https://lore.kernel.org/r/20221027155908.1940624-3-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Makefile | 2 ++
arch/Kconfig | 7 +++++++
include/linux/scs.h | 18 ++++++++++++++++++
kernel/scs.c | 14 ++++++++++++--
4 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index ac2ec990422d..69b40b629f3c 100644
--- a/Makefile
+++ b/Makefile
@@ -966,8 +966,10 @@ LDFLAGS_vmlinux += --gc-sections
endif
ifdef CONFIG_SHADOW_CALL_STACK
+ifndef CONFIG_DYNAMIC_SCS
CC_FLAGS_SCS := -fsanitize=shadow-call-stack
KBUILD_CFLAGS += $(CC_FLAGS_SCS)
+endif
export CC_FLAGS_SCS
endif
diff --git a/arch/Kconfig b/arch/Kconfig
index 8f138e580d1a..072a1b39e3af 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -651,6 +651,13 @@ config SHADOW_CALL_STACK
reading and writing arbitrary memory may be able to locate them
and hijack control flow by modifying the stacks.
+config DYNAMIC_SCS
+ bool
+ help
+ Set by the arch code if it relies on code patching to insert the
+ shadow call stack push and pop instructions rather than on the
+ compiler.
+
config LTO
bool
help
diff --git a/include/linux/scs.h b/include/linux/scs.h
index 18122d9e17ff..4ab5bdc898cf 100644
--- a/include/linux/scs.h
+++ b/include/linux/scs.h
@@ -53,6 +53,22 @@ static inline bool task_scs_end_corrupted(struct task_struct *tsk)
return sz >= SCS_SIZE - 1 || READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
}
+DECLARE_STATIC_KEY_FALSE(dynamic_scs_enabled);
+
+static inline bool scs_is_dynamic(void)
+{
+ if (!IS_ENABLED(CONFIG_DYNAMIC_SCS))
+ return false;
+ return static_branch_likely(&dynamic_scs_enabled);
+}
+
+static inline bool scs_is_enabled(void)
+{
+ if (!IS_ENABLED(CONFIG_DYNAMIC_SCS))
+ return true;
+ return scs_is_dynamic();
+}
+
#else /* CONFIG_SHADOW_CALL_STACK */
static inline void *scs_alloc(int node) { return NULL; }
@@ -62,6 +78,8 @@ static inline void scs_task_reset(struct task_struct *tsk) {}
static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
static inline void scs_release(struct task_struct *tsk) {}
static inline bool task_scs_end_corrupted(struct task_struct *tsk) { return false; }
+static inline bool scs_is_enabled(void) { return false; }
+static inline bool scs_is_dynamic(void) { return false; }
#endif /* CONFIG_SHADOW_CALL_STACK */
diff --git a/kernel/scs.c b/kernel/scs.c
index b7e1b096d906..d7809affe740 100644
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -12,6 +12,10 @@
#include <linux/vmalloc.h>
#include <linux/vmstat.h>
+#ifdef CONFIG_DYNAMIC_SCS
+DEFINE_STATIC_KEY_FALSE(dynamic_scs_enabled);
+#endif
+
static void __scs_account(void *s, int account)
{
struct page *scs_page = vmalloc_to_page(s);
@@ -101,14 +105,20 @@ static int scs_cleanup(unsigned int cpu)
void __init scs_init(void)
{
+ if (!scs_is_enabled())
+ return;
cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
scs_cleanup);
}
int scs_prepare(struct task_struct *tsk, int node)
{
- void *s = scs_alloc(node);
+ void *s;
+ if (!scs_is_enabled())
+ return 0;
+
+ s = scs_alloc(node);
if (!s)
return -ENOMEM;
@@ -148,7 +158,7 @@ void scs_release(struct task_struct *tsk)
{
void *s = task_scs(tsk);
- if (!s)
+ if (!scs_is_enabled() || !s)
return;
WARN(task_scs_end_corrupted(tsk),
From 3b619e22c4601b444ed2d6a5458271f72625ac89 Mon Sep 17 00:00:00 2001
From: Ard Biesheuvel <ardb@kernel.org>
Date: Thu, 27 Oct 2022 17:59:08 +0200
Subject: [PATCH 029/151] arm64: implement dynamic shadow call stack for Clang
Implement dynamic shadow call stack support on Clang, by parsing the
unwind tables at init time to locate all occurrences of PACIASP/AUTIASP
instructions, and replacing them with the shadow call stack push and pop
instructions, respectively.
This is useful because the overhead of the shadow call stack is
difficult to justify on hardware that implements pointer authentication
(PAC), and given that the PAC instructions are executed as NOPs on
hardware that doesn't, we can just replace them without breaking
anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to
manipulations of the return address, replacing them 1:1 with shadow call
stack pushes and pops is guaranteed to result in the desired behavior.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Sami Tolvanen <samitolvanen@google.com>
Tested-by: Sami Tolvanen <samitolvanen@google.com>
Link: https://lore.kernel.org/r/20221027155908.1940624-4-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 9 ++
arch/arm64/Makefile | 10 +-
arch/arm64/include/asm/scs.h | 49 +++++++
arch/arm64/kernel/Makefile | 2 +
arch/arm64/kernel/head.S | 3 +
arch/arm64/kernel/irq.c | 2 +-
arch/arm64/kernel/module.c | 8 ++
arch/arm64/kernel/patch-scs.c | 257 ++++++++++++++++++++++++++++++++++
arch/arm64/kernel/sdei.c | 2 +-
arch/arm64/kernel/setup.c | 4 +
10 files changed, 342 insertions(+), 4 deletions(-)
create mode 100644 arch/arm64/kernel/patch-scs.c
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 7e3a9cf2193d..170832f31eff 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2160,6 +2160,15 @@ config ARCH_NR_GPIO
If unsure, leave the default value.
+config UNWIND_PATCH_PAC_INTO_SCS
+ bool "Enable shadow call stack dynamically using code patching"
+ # needs Clang with https://reviews.llvm.org/D111780 incorporated
+ depends on CC_IS_CLANG && CLANG_VERSION >= 150000
+ depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET
+ depends on SHADOW_CALL_STACK
+ select UNWIND_TABLES
+ select DYNAMIC_SCS
+
endmenu # "Kernel Features"
menu "Boot options"
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 7868a176993f..4d272ad1df1f 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -77,10 +77,16 @@ branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=
# We enable additional protection for leaf functions as there is some
# narrow potential for ROP protection benefits and no substantial
# performance impact has been observed.
+PACRET-y := pac-ret+leaf
+
+# Using a shadow call stack in leaf functions is too costly, so avoid PAC there
+# as well when we may be patching PAC into SCS
+PACRET-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) := pac-ret
+
ifeq ($(CONFIG_ARM64_BTI_KERNEL),y)
-branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti
+branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=$(PACRET-y)+bti
else
-branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf
+branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=$(PACRET-y)
endif
# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the
# compiler to generate them and consequently to break the single image contract
diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 8297bccf0784..ff7da1268a52 100644
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -5,6 +5,7 @@
#ifdef __ASSEMBLY__
#include <asm/asm-offsets.h>
+#include <asm/sysreg.h>
#ifdef CONFIG_SHADOW_CALL_STACK
scs_sp .req x18
@@ -24,6 +25,54 @@
.endm
#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#else
+
+#include <linux/scs.h>
+#include <asm/cpufeature.h>
+
+#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
+static inline bool should_patch_pac_into_scs(void)
+{
+ u64 reg;
+
+ /*
+ * We only enable the shadow call stack dynamically if we are running
+ * on a system that does not implement PAC or BTI. PAC and SCS provide
+ * roughly the same level of protection, and BTI relies on the PACIASP
+ * instructions serving as landing pads, preventing us from patching
+ * those instructions into something else.
+ */
+ reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1);
+ if (SYS_FIELD_GET(ID_AA64ISAR1_EL1, APA, reg) |
+ SYS_FIELD_GET(ID_AA64ISAR1_EL1, API, reg))
+ return false;
+
+ reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
+ if (SYS_FIELD_GET(ID_AA64ISAR2_EL1, APA3, reg))
+ return false;
+
+ if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {
+ reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1);
+ if (reg & (0xf << ID_AA64PFR1_EL1_BT_SHIFT))
+ return false;
+ }
+ return true;
+}
+
+static inline void dynamic_scs_init(void)
+{
+ if (should_patch_pac_into_scs()) {
+ pr_info("Enabling dynamic shadow call stack\n");
+ static_branch_enable(&dynamic_scs_enabled);
+ }
+}
+#else
+static inline void dynamic_scs_init(void) {}
+#endif
+
+int scs_patch(const u8 eh_frame[], int size);
+
#endif /* __ASSEMBLY __ */
#endif /* _ASM_SCS_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 2f361a883d8c..8dd925f4a4c6 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -80,6 +80,8 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
obj-$(CONFIG_ARM64_MTE) += mte.o
obj-y += vdso-wrap.o
obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o
+obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o
+CFLAGS_patch-scs.o += -mbranch-protection=none
# Force dependency (vdso*-wrap.S includes vdso.so through incbin)
$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 2196aad7b55b..952e17bd1c0b 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -462,6 +462,9 @@ SYM_FUNC_START_LOCAL(__primary_switched)
bl early_fdt_map // Try mapping the FDT early
mov x0, x20 // pass the full boot status
bl init_feature_override // Parse cpu feature overrides
+#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
+ bl scs_patch_vmlinux
+#endif
mov x0, x20
bl finalise_el2 // Prefer VHE if possible
ldp x29, x30, [sp], #16
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 38dbd3828f13..9d8eaab742a4 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -41,7 +41,7 @@ static void init_irq_scs(void)
{
int cpu;
- if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
+ if (!scs_is_enabled())
return;
for_each_possible_cpu(cpu)
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 76b41e4ca9fa..fa7b3228944b 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -15,9 +15,11 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/moduleloader.h>
+#include <linux/scs.h>
#include <linux/vmalloc.h>
#include <asm/alternative.h>
#include <asm/insn.h>
+#include <asm/scs.h>
#include <asm/sections.h>
void *module_alloc(unsigned long size)
@@ -514,5 +516,11 @@ int module_finalize(const Elf_Ehdr *hdr,
if (s)
apply_alternatives_module((void *)s->sh_addr, s->sh_size);
+ if (scs_is_dynamic()) {
+ s = find_section(hdr, sechdrs, ".init.eh_frame");
+ if (s)
+ scs_patch((void *)s->sh_addr, s->sh_size);
+ }
+
return module_init_ftrace_plt(hdr, sechdrs, me);
}
diff --git a/arch/arm64/kernel/patch-scs.c b/arch/arm64/kernel/patch-scs.c
new file mode 100644
index 000000000000..1b3da02d5b74
--- /dev/null
+++ b/arch/arm64/kernel/patch-scs.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 - Google LLC
+ * Author: Ard Biesheuvel <ardb@google.com>
+ */
+
+#include <linux/bug.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/printk.h>
+#include <linux/types.h>
+
+#include <asm/cacheflush.h>
+#include <asm/scs.h>
+
+//
+// This minimal DWARF CFI parser is partially based on the code in
+// arch/arc/kernel/unwind.c, and on the document below:
+// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
+//
+
+#define DW_CFA_nop 0x00
+#define DW_CFA_set_loc 0x01
+#define DW_CFA_advance_loc1 0x02
+#define DW_CFA_advance_loc2 0x03
+#define DW_CFA_advance_loc4 0x04
+#define DW_CFA_offset_extended 0x05
+#define DW_CFA_restore_extended 0x06
+#define DW_CFA_undefined 0x07
+#define DW_CFA_same_value 0x08
+#define DW_CFA_register 0x09
+#define DW_CFA_remember_state 0x0a
+#define DW_CFA_restore_state 0x0b
+#define DW_CFA_def_cfa 0x0c
+#define DW_CFA_def_cfa_register 0x0d
+#define DW_CFA_def_cfa_offset 0x0e
+#define DW_CFA_def_cfa_expression 0x0f
+#define DW_CFA_expression 0x10
+#define DW_CFA_offset_extended_sf 0x11
+#define DW_CFA_def_cfa_sf 0x12
+#define DW_CFA_def_cfa_offset_sf 0x13
+#define DW_CFA_val_offset 0x14
+#define DW_CFA_val_offset_sf 0x15
+#define DW_CFA_val_expression 0x16
+#define DW_CFA_lo_user 0x1c
+#define DW_CFA_negate_ra_state 0x2d
+#define DW_CFA_GNU_args_size 0x2e
+#define DW_CFA_GNU_negative_offset_extended 0x2f
+#define DW_CFA_hi_user 0x3f
+
+extern const u8 __eh_frame_start[], __eh_frame_end[];
+
+enum {
+ PACIASP = 0xd503233f,
+ AUTIASP = 0xd50323bf,
+ SCS_PUSH = 0xf800865e,
+ SCS_POP = 0xf85f8e5e,
+};
+
+static void __always_inline scs_patch_loc(u64 loc)
+{
+ u32 insn = le32_to_cpup((void *)loc);
+
+ switch (insn) {
+ case PACIASP:
+ *(u32 *)loc = cpu_to_le32(SCS_PUSH);
+ break;
+ case AUTIASP:
+ *(u32 *)loc = cpu_to_le32(SCS_POP);
+ break;
+ default:
+ /*
+ * While the DW_CFA_negate_ra_state directive is guaranteed to
+ * appear right after a PACIASP/AUTIASP instruction, it may
+ * also appear after a DW_CFA_restore_state directive that
+ * restores a state that is only partially accurate, and is
+ * followed by DW_CFA_negate_ra_state directive to toggle the
+ * PAC bit again. So we permit other instructions here, and ignore
+ * them.
+ */
+ return;
+ }
+ dcache_clean_pou(loc, loc + sizeof(u32));
+}
+
+/*
+ * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
+ * except the last one have bit #7 set.
+ */
+static int __always_inline skip_xleb128(const u8 **opcode, int size)
+{
+ u8 c;
+
+ do {
+ c = *(*opcode)++;
+ size--;
+ } while (c & BIT(7));
+
+ return size;
+}
+
+struct eh_frame {
+ /*
+ * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
+ */
+ u32 size;
+
+ /*
+ * The first frame is a Common Information Entry (CIE) frame, followed
+ * by one or more Frame Description Entry (FDE) frames. In the former
+ * case, this field is 0, otherwise it is the negated offset relative
+ * to the associated CIE frame.
+ */
+ u32 cie_id_or_pointer;
+
+ union {
+ struct { // CIE
+ u8 version;
+ u8 augmentation_string[];
+ };
+
+ struct { // FDE
+ s32 initial_loc;
+ s32 range;
+ u8 opcodes[];
+ };
+ };
+};
+
+static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
+ bool fde_has_augmentation_data,
+ int code_alignment_factor)
+{
+ int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
+ u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
+ const u8 *opcode = frame->opcodes;
+
+ if (fde_has_augmentation_data) {
+ int l;
+
+ // assume single byte uleb128_t
+ if (WARN_ON(*opcode & BIT(7)))
+ return -ENOEXEC;
+
+ l = *opcode++;
+ opcode += l;
+ size -= l + 1;
+ }
+
+ /*
+ * Starting from 'loc', apply the CFA opcodes that advance the location
+ * pointer, and identify the locations of the PAC instructions.
+ */
+ while (size-- > 0) {
+ switch (*opcode++) {
+ case DW_CFA_nop:
+ case DW_CFA_remember_state:
+ case DW_CFA_restore_state:
+ break;
+
+ case DW_CFA_advance_loc1:
+ loc += *opcode++ * code_alignment_factor;
+ size--;
+ break;
+
+ case DW_CFA_advance_loc2:
+ loc += *opcode++ * code_alignment_factor;
+ loc += (*opcode++ << 8) * code_alignment_factor;
+ size -= 2;
+ break;
+
+ case DW_CFA_def_cfa:
+ case DW_CFA_offset_extended:
+ size = skip_xleb128(&opcode, size);
+ fallthrough;
+ case DW_CFA_def_cfa_offset:
+ case DW_CFA_def_cfa_offset_sf:
+ case DW_CFA_def_cfa_register:
+ case DW_CFA_same_value:
+ case DW_CFA_restore_extended:
+ case 0x80 ... 0xbf:
+ size = skip_xleb128(&opcode, size);
+ break;
+
+ case DW_CFA_negate_ra_state:
+ scs_patch_loc(loc - 4);
+ break;
+
+ case 0x40 ... 0x7f:
+ // advance loc
+ loc += (opcode[-1] & 0x3f) * code_alignment_factor;
+ break;
+
+ case 0xc0 ... 0xff:
+ break;
+
+ default:
+ pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
+ return -ENOEXEC;
+ }
+ }
+ return 0;
+}
+
+int noinstr scs_patch(const u8 eh_frame[], int size)
+{
+ const u8 *p = eh_frame;
+
+ while (size > 4) {
+ const struct eh_frame *frame = (const void *)p;
+ bool fde_has_augmentation_data = true;
+ int code_alignment_factor = 1;
+ int ret;
+
+ if (frame->size == 0 ||
+ frame->size == U32_MAX ||
+ frame->size > size)
+ break;
+
+ if (frame->cie_id_or_pointer == 0) {
+ const u8 *p = frame->augmentation_string;
+
+ /* a 'z' in the augmentation string must come first */
+ fde_has_augmentation_data = *p == 'z';
+
+ /*
+ * The code alignment factor is a uleb128 encoded field
+ * but given that the only sensible values are 1 or 4,
+ * there is no point in decoding the whole thing.
+ */
+ p += strlen(p) + 1;
+ if (!WARN_ON(*p & BIT(7)))
+ code_alignment_factor = *p;
+ } else {
+ ret = scs_handle_fde_frame(frame,
+ fde_has_augmentation_data,
+ code_alignment_factor);
+ if (ret)
+ return ret;
+ }
+
+ p += sizeof(frame->size) + frame->size;
+ size -= sizeof(frame->size) + frame->size;
+ }
+ return 0;
+}
+
+asmlinkage void __init scs_patch_vmlinux(void)
+{
+ if (!should_patch_pac_into_scs())
+ return;
+
+ WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
+ icache_inval_all_pou();
+ isb();
+}
diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c
index d56e170e1ca7..830be01af32d 100644
--- a/arch/arm64/kernel/sdei.c
+++ b/arch/arm64/kernel/sdei.c
@@ -144,7 +144,7 @@ static int init_sdei_scs(void)
int cpu;
int err = 0;
- if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
+ if (!scs_is_enabled())
return 0;
for_each_possible_cpu(cpu) {
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index fea3223704b6..12cfe9d0d3fa 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -30,6 +30,7 @@
#include <linux/efi.h>
#include <linux/psci.h>
#include <linux/sched/task.h>
+#include <linux/scs.h>
#include <linux/mm.h>
#include <asm/acpi.h>
@@ -42,6 +43,7 @@
#include <asm/cpu_ops.h>
#include <asm/kasan.h>
#include <asm/numa.h>
+#include <asm/scs.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/smp_plat.h>
@@ -312,6 +314,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
jump_label_init();
parse_early_param();
+ dynamic_scs_init();
+
/*
* Unmask asynchronous aborts and fiq after bringing up possible
* earlycon. (Report possible System Errors once we can report this
From 1e55b44d9ecd42dee7106ce635ffd855183f5221 Mon Sep 17 00:00:00 2001
From: Usama Arif <usama.arif@bytedance.com>
Date: Fri, 4 Nov 2022 06:16:59 +0000
Subject: [PATCH 030/151] arm64: paravirt: remove conduit check in
has_pv_steal_clock
arm_smccc_1_1_invoke() which is called later on in the function
will return failure if there's no conduit (or pre-SMCCC 1.1),
hence the check is unnecessary.
Suggested-by: Steven Price <steven.price@arm.com>
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
Reviewed-by: Steven Price <steven.price@arm.com>
Link: https://lore.kernel.org/r/20221104061659.4116508-1-usama.arif@bytedance.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/paravirt.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c
index 57c7c211f8c7..aa718d6a9274 100644
--- a/arch/arm64/kernel/paravirt.c
+++ b/arch/arm64/kernel/paravirt.c
@@ -141,10 +141,6 @@ static bool __init has_pv_steal_clock(void)
{
struct arm_smccc_res res;
- /* To detect the presence of PV time support we require SMCCC 1.1+ */
- if (arm_smccc_1_1_get_conduit() == SMCCC_CONDUIT_NONE)
- return false;
-
arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_HV_PV_TIME_FEATURES, &res);
From a4ee28615c7a1e2925e1fcb4ba0fa1aeee633d78 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Mon, 7 Nov 2022 19:47:53 +0530
Subject: [PATCH 031/151] arm64/mm: Simplify and document pte_to_phys() for 52
bit addresses
pte_to_phys() assembly definition does multiple bits field transformations
to derive physical address, embedded inside a page table entry. Unlike its
C counter part i.e __pte_to_phys(), pte_to_phys() is not very apparent. It
simplifies these operations via a new macro PTE_ADDR_HIGH_SHIFT indicating
how far the pte encoded higher address bits need to be left shifted. While
here, this also updates __pte_to_phys() and __phys_to_pte_val().
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Brown <broonie@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Suggested-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lore.kernel.org/r/20221107141753.2938621-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/assembler.h | 8 +++-----
arch/arm64/include/asm/pgtable-hwdef.h | 1 +
arch/arm64/include/asm/pgtable.h | 4 ++--
3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index e5957a53be39..89038067ef34 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -660,12 +660,10 @@ alternative_endif
.endm
.macro pte_to_phys, phys, pte
-#ifdef CONFIG_ARM64_PA_BITS_52
- ubfiz \phys, \pte, #(48 - 16 - 12), #16
- bfxil \phys, \pte, #16, #32
- lsl \phys, \phys, #16
-#else
and \phys, \pte, #PTE_ADDR_MASK
+#ifdef CONFIG_ARM64_PA_BITS_52
+ orr \phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT
+ and \phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT)
#endif
.endm
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 5ab8d163198f..f658aafc47df 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -159,6 +159,7 @@
#ifdef CONFIG_ARM64_PA_BITS_52
#define PTE_ADDR_HIGH (_AT(pteval_t, 0xf) << 12)
#define PTE_ADDR_MASK (PTE_ADDR_LOW | PTE_ADDR_HIGH)
+#define PTE_ADDR_HIGH_SHIFT 36
#else
#define PTE_ADDR_MASK PTE_ADDR_LOW
#endif
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 71a1af42f0e8..daedd6172227 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -77,11 +77,11 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
static inline phys_addr_t __pte_to_phys(pte_t pte)
{
return (pte_val(pte) & PTE_ADDR_LOW) |
- ((pte_val(pte) & PTE_ADDR_HIGH) << 36);
+ ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
}
static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
{
- return (phys | (phys >> 36)) & PTE_ADDR_MASK;
+ return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK;
}
#else
#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
From a75df5be8e7b0ecfb413b7c02a2599be17d2721b Mon Sep 17 00:00:00 2001
From: "wangkailong@jari.cn" <wangkailong@jari.cn>
Date: Sun, 13 Nov 2022 17:41:10 +0800
Subject: [PATCH 032/151] kselftest/arm64: fix array_size.cocci warning
Fix following coccicheck warning:
tools/testing/selftests/arm64/mte/check_mmap_options.c:64:24-25:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_mmap_options.c:66:20-21:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_mmap_options.c:135:25-26:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_mmap_options.c:96:25-26:
WARNING: Use ARRAY_SIZE
tools/testing/selftests/arm64/mte/check_mmap_options.c:190:24-25:
WARNING: Use ARRAY_SIZE
Signed-off-by: KaiLong Wang <wangkailong@jari.cn>
Link: https://lore.kernel.org/r/777ce8ba.12e.184705d4211.Coremail.wangkailong@jari.cn
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/mte/check_mmap_options.c | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/tools/testing/selftests/arm64/mte/check_mmap_options.c b/tools/testing/selftests/arm64/mte/check_mmap_options.c
index a04b12c21ac9..17694caaff53 100644
--- a/tools/testing/selftests/arm64/mte/check_mmap_options.c
+++ b/tools/testing/selftests/arm64/mte/check_mmap_options.c
@@ -61,9 +61,8 @@ static int check_anonymous_memory_mapping(int mem_type, int mode, int mapping, i
{
char *ptr, *map_ptr;
int run, result, map_size;
- int item = sizeof(sizes)/sizeof(int);
+ int item = ARRAY_SIZE(sizes);
- item = sizeof(sizes)/sizeof(int);
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
for (run = 0; run < item; run++) {
map_size = sizes[run] + OVERFLOW + UNDERFLOW;
@@ -93,7 +92,7 @@ static int check_file_memory_mapping(int mem_type, int mode, int mapping, int ta
{
char *ptr, *map_ptr;
int run, fd, map_size;
- int total = sizeof(sizes)/sizeof(int);
+ int total = ARRAY_SIZE(sizes);
int result = KSFT_PASS;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
@@ -132,7 +131,7 @@ static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping)
{
char *ptr, *map_ptr;
int run, prot_flag, result, fd, map_size;
- int total = sizeof(sizes)/sizeof(int);
+ int total = ARRAY_SIZE(sizes);
prot_flag = PROT_READ | PROT_WRITE;
mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
@@ -187,7 +186,7 @@ static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping)
int main(int argc, char *argv[])
{
int err;
- int item = sizeof(sizes)/sizeof(int);
+ int item = ARRAY_SIZE(sizes);
err = mte_default_setup();
if (err)
From 657eef0a5420a02c02945ed8c87f2ddcbd255772 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 12:54:24 +0000
Subject: [PATCH 033/151] arm64: atomics: lse: remove stale dependency on
JUMP_LABEL
Currently CONFIG_ARM64_USE_LSE_ATOMICS depends upon CONFIG_JUMP_LABEL,
as the inline atomics were indirected with a static branch.
However, since commit:
21fb26bfb01ffe0d ("arm64: alternatives: add alternative_has_feature_*()")
... we use an alternative_branch (which is always available) rather than
a static branch, and hence the dependency is unnecessary.
Remove the stale dependency, along with the stale include. This will
allow the use of LSE atomics in kernels built with CONFIG_JUMP_LABEL=n,
and reduces the risk of circular header dependencies via <asm/lse.h>.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221114125424.2998268-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 1 -
arch/arm64/include/asm/lse.h | 1 -
2 files changed, 2 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 505c8a1ccbe0..6a4704d4e36b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1714,7 +1714,6 @@ config ARM64_LSE_ATOMICS
config ARM64_USE_LSE_ATOMICS
bool "Atomic instructions"
- depends on JUMP_LABEL
default y
help
As part of the Large System Extensions, ARMv8.1 introduces new
diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h
index c503db8e73b0..f99d74826a7e 100644
--- a/arch/arm64/include/asm/lse.h
+++ b/arch/arm64/include/asm/lse.h
@@ -10,7 +10,6 @@
#include <linux/compiler_types.h>
#include <linux/export.h>
-#include <linux/jump_label.h>
#include <linux/stringify.h>
#include <asm/alternative.h>
#include <asm/alternative-macros.h>
From 9e75e74b07ab17e6407b567eb4a8b3d0c93de5f1 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Mon, 31 Oct 2022 19:24:50 +0000
Subject: [PATCH 034/151] arm64/signal: Document our convention for choosing
magic numbers
Szabolcs Nagy has pointed out that most of our signal frame magic numbers
are chosen to be meaningful ASCII when dumped to aid manual parsing. This
seems sensible since it might help someone parsing things out, let's
document it so people implementing new signal contexts are aware of it and
are more likely to follow it.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221031192450.826159-1-broonie@kernel.org
[will: Fixed typo and tweaked wording]
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/uapi/asm/sigcontext.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h
index 4aaf31e3bf16..9525041e4a14 100644
--- a/arch/arm64/include/uapi/asm/sigcontext.h
+++ b/arch/arm64/include/uapi/asm/sigcontext.h
@@ -62,6 +62,10 @@ struct sigcontext {
* context. Such structures must be placed after the rt_sigframe on the stack
* and be 16-byte aligned. The last structure must be a dummy one with the
* magic and size set to 0.
+ *
+ * Note that the values allocated for use as magic should be chosen to
+ * be meaningful in ASCII to aid manual parsing, ZA doesn't follow this
+ * convention due to oversight but it should be observed for future additions.
*/
struct _aarch64_ctx {
__u32 magic;
From c2465f95c4e73af1a6564b00ebc9acc15485edf0 Mon Sep 17 00:00:00 2001
From: Jeremy Linton <jeremy.linton@arm.com>
Date: Wed, 9 Nov 2022 11:47:20 -0600
Subject: [PATCH 035/151] ACPI: Enable FPDT on arm64
FPDT provides some boot timing records useful for analyzing
parts of the UEFI boot stack. Given the existing code works
on arm64, and allows reading the values without utilizing
/dev/mem it seems like a good idea to turn it on.
Signed-off-by: Jeremy Linton <jeremy.linton@arm.com>
Acked-by: Sudeep Holla <sudeep.holla@arm.com>
Link: https://lore.kernel.org/r/20221109174720.203723-1-jeremy.linton@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/acpi_object_usage.rst | 2 +-
drivers/acpi/Kconfig | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/arm64/acpi_object_usage.rst b/Documentation/arm64/acpi_object_usage.rst
index 0609da73970b..484ef9676653 100644
--- a/Documentation/arm64/acpi_object_usage.rst
+++ b/Documentation/arm64/acpi_object_usage.rst
@@ -163,7 +163,7 @@ FPDT Section 5.2.23 (signature == "FPDT")
**Firmware Performance Data Table**
- Optional, not currently supported.
+ Optional, useful for boot performance profiling.
GTDT Section 5.2.24 (signature == "GTDT")
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 473241b5193f..1cc11d2a2a88 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -90,7 +90,7 @@ config ACPI_SPCR_TABLE
config ACPI_FPDT
bool "ACPI Firmware Performance Data Table (FPDT) support"
- depends on X86_64
+ depends on X86_64 || ARM64
help
Enable support for the Firmware Performance Data Table (FPDT).
This table provides information on the timing of the system
From d8c1d798a2e5091128c391c6dadcc9be334af3f5 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 14:40:42 +0000
Subject: [PATCH 036/151] arm64: make is_ttbrX_addr() noinstr-safe
We use is_ttbr0_addr() in noinstr code, but as it's only marked as
inline, it's theoretically possible for the compiler to place it
out-of-line and instrument it, which would be problematic.
Mark is_ttbr0_addr() as __always_inline such that that can safely be
used from noinstr code. For consistency, do the same to is_ttbr1_addr().
Note that while is_ttbr1_addr() calls arch_kasan_reset_tag(), this is a
macro (and its callees are either macros or __always_inline), so there
is not a risk of transient instrumentation.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221114144042.3001140-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/processor.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 445aa3af3b76..400f8956328b 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -308,13 +308,13 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
}
#endif
-static inline bool is_ttbr0_addr(unsigned long addr)
+static __always_inline bool is_ttbr0_addr(unsigned long addr)
{
/* entry assembly clears tags for TTBR0 addrs */
return addr < TASK_SIZE;
}
-static inline bool is_ttbr1_addr(unsigned long addr)
+static __always_inline bool is_ttbr1_addr(unsigned long addr)
{
/* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
From 1b6bf2da7b1d2ba7560c793b22d66f134ac61416 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 13:59:25 +0000
Subject: [PATCH 037/151] arm64: insn: remove aarch64_insn_gen_prefetch()
There are no users of aarch64_insn_gen_prefetch(), and which encodes a
PRFM (immediate) with a hard-coded offset of 0.
Remove it for now; we can always restore it with tests if we need it in
future.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Joey Gouly <joey.gouly@arm.com>
Link: https://lore.kernel.org/r/20221114135928.3000571-2-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/insn.h | 4 --
arch/arm64/lib/insn.c | 70 -----------------------------------
2 files changed, 74 deletions(-)
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 834bff720582..03f60234e60a 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -580,10 +580,6 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
enum aarch64_insn_register Rn,
enum aarch64_insn_register Rd,
u8 lsb);
-u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
- enum aarch64_insn_prfm_type type,
- enum aarch64_insn_prfm_target target,
- enum aarch64_insn_prfm_policy policy);
#ifdef CONFIG_ARM64_LSE_ATOMICS
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
enum aarch64_insn_register address,
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index 49e972beeac7..ac0df03fffcd 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -816,76 +816,6 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
}
#endif
-static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
- enum aarch64_insn_prfm_target target,
- enum aarch64_insn_prfm_policy policy,
- u32 insn)
-{
- u32 imm_type = 0, imm_target = 0, imm_policy = 0;
-
- switch (type) {
- case AARCH64_INSN_PRFM_TYPE_PLD:
- break;
- case AARCH64_INSN_PRFM_TYPE_PLI:
- imm_type = BIT(0);
- break;
- case AARCH64_INSN_PRFM_TYPE_PST:
- imm_type = BIT(1);
- break;
- default:
- pr_err("%s: unknown prfm type encoding %d\n", __func__, type);
- return AARCH64_BREAK_FAULT;
- }
-
- switch (target) {
- case AARCH64_INSN_PRFM_TARGET_L1:
- break;
- case AARCH64_INSN_PRFM_TARGET_L2:
- imm_target = BIT(0);
- break;
- case AARCH64_INSN_PRFM_TARGET_L3:
- imm_target = BIT(1);
- break;
- default:
- pr_err("%s: unknown prfm target encoding %d\n", __func__, target);
- return AARCH64_BREAK_FAULT;
- }
-
- switch (policy) {
- case AARCH64_INSN_PRFM_POLICY_KEEP:
- break;
- case AARCH64_INSN_PRFM_POLICY_STRM:
- imm_policy = BIT(0);
- break;
- default:
- pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy);
- return AARCH64_BREAK_FAULT;
- }
-
- /* In this case, imm5 is encoded into Rt field. */
- insn &= ~GENMASK(4, 0);
- insn |= imm_policy | (imm_target << 1) | (imm_type << 3);
-
- return insn;
-}
-
-u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
- enum aarch64_insn_prfm_type type,
- enum aarch64_insn_prfm_target target,
- enum aarch64_insn_prfm_policy policy)
-{
- u32 insn = aarch64_insn_get_prfm_value();
-
- insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn);
-
- insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn);
-
- insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
- base);
-
- return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0);
-}
-
u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
int imm, enum aarch64_insn_variant variant,
From 9b948e79d5369bda7a628bf69521172200d07613 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 13:59:26 +0000
Subject: [PATCH 038/151] arm64: insn: always inline predicates
We have a number of aarch64_insn_*() predicates which are used in code
which is not instrumentation safe (e.g. alternatives patching, kprobes).
Some of those are marked with __kprobes, but most are not, and are
implemented out-of-line in insn.c.
This patch moves the predicates to insn.h and marks them with
__always_inline. This is ensures that they will respect the
instrumentation requirements of their caller which they will be inlined
into.
At the same time, I've formatted each of the functions consistently as a
list, to make them easier to read and update in future.
Other than preventing unwanted instrumentation, there should be no
functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Joey Gouly <joey.gouly@arm.com>
Link: https://lore.kernel.org/r/20221114135928.3000571-3-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/insn.h | 98 +++++++++++++++++++++++++++++------
arch/arm64/lib/insn.c | 61 ----------------------
2 files changed, 81 insertions(+), 78 deletions(-)
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 03f60234e60a..de5bc71b4b1d 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -431,58 +431,122 @@ __AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F)
#undef __AARCH64_INSN_FUNCS
-bool aarch64_insn_is_steppable_hint(u32 insn);
-bool aarch64_insn_is_branch_imm(u32 insn);
-
-static inline bool aarch64_insn_is_adr_adrp(u32 insn)
+static __always_inline bool aarch64_insn_is_steppable_hint(u32 insn)
{
- return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn);
+ if (!aarch64_insn_is_hint(insn))
+ return false;
+
+ switch (insn & 0xFE0) {
+ case AARCH64_INSN_HINT_XPACLRI:
+ case AARCH64_INSN_HINT_PACIA_1716:
+ case AARCH64_INSN_HINT_PACIB_1716:
+ case AARCH64_INSN_HINT_PACIAZ:
+ case AARCH64_INSN_HINT_PACIASP:
+ case AARCH64_INSN_HINT_PACIBZ:
+ case AARCH64_INSN_HINT_PACIBSP:
+ case AARCH64_INSN_HINT_BTI:
+ case AARCH64_INSN_HINT_BTIC:
+ case AARCH64_INSN_HINT_BTIJ:
+ case AARCH64_INSN_HINT_BTIJC:
+ case AARCH64_INSN_HINT_NOP:
+ return true;
+ default:
+ return false;
+ }
}
-static inline bool aarch64_insn_is_dsb(u32 insn)
+static __always_inline bool aarch64_insn_is_branch(u32 insn)
{
- return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn);
+ /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
+
+ return aarch64_insn_is_b(insn) ||
+ aarch64_insn_is_bl(insn) ||
+ aarch64_insn_is_cbz(insn) ||
+ aarch64_insn_is_cbnz(insn) ||
+ aarch64_insn_is_tbz(insn) ||
+ aarch64_insn_is_tbnz(insn) ||
+ aarch64_insn_is_ret(insn) ||
+ aarch64_insn_is_ret_auth(insn) ||
+ aarch64_insn_is_br(insn) ||
+ aarch64_insn_is_br_auth(insn) ||
+ aarch64_insn_is_blr(insn) ||
+ aarch64_insn_is_blr_auth(insn) ||
+ aarch64_insn_is_bcond(insn);
}
-static inline bool aarch64_insn_is_barrier(u32 insn)
+static __always_inline bool aarch64_insn_is_branch_imm(u32 insn)
{
- return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) ||
- aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) ||
- aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) ||
+ return aarch64_insn_is_b(insn) ||
+ aarch64_insn_is_bl(insn) ||
+ aarch64_insn_is_tbz(insn) ||
+ aarch64_insn_is_tbnz(insn) ||
+ aarch64_insn_is_cbz(insn) ||
+ aarch64_insn_is_cbnz(insn) ||
+ aarch64_insn_is_bcond(insn);
+}
+
+static __always_inline bool aarch64_insn_is_adr_adrp(u32 insn)
+{
+ return aarch64_insn_is_adr(insn) ||
+ aarch64_insn_is_adrp(insn);
+}
+
+static __always_inline bool aarch64_insn_is_dsb(u32 insn)
+{
+ return aarch64_insn_is_dsb_base(insn) ||
+ aarch64_insn_is_dsb_nxs(insn);
+}
+
+static __always_inline bool aarch64_insn_is_barrier(u32 insn)
+{
+ return aarch64_insn_is_dmb(insn) ||
+ aarch64_insn_is_dsb(insn) ||
+ aarch64_insn_is_isb(insn) ||
+ aarch64_insn_is_sb(insn) ||
+ aarch64_insn_is_clrex(insn) ||
+ aarch64_insn_is_ssbb(insn) ||
aarch64_insn_is_pssbb(insn);
}
-static inline bool aarch64_insn_is_store_single(u32 insn)
+static __always_inline bool aarch64_insn_is_store_single(u32 insn)
{
return aarch64_insn_is_store_imm(insn) ||
aarch64_insn_is_store_pre(insn) ||
aarch64_insn_is_store_post(insn);
}
-static inline bool aarch64_insn_is_store_pair(u32 insn)
+static __always_inline bool aarch64_insn_is_store_pair(u32 insn)
{
return aarch64_insn_is_stp(insn) ||
aarch64_insn_is_stp_pre(insn) ||
aarch64_insn_is_stp_post(insn);
}
-static inline bool aarch64_insn_is_load_single(u32 insn)
+static __always_inline bool aarch64_insn_is_load_single(u32 insn)
{
return aarch64_insn_is_load_imm(insn) ||
aarch64_insn_is_load_pre(insn) ||
aarch64_insn_is_load_post(insn);
}
-static inline bool aarch64_insn_is_load_pair(u32 insn)
+static __always_inline bool aarch64_insn_is_load_pair(u32 insn)
{
return aarch64_insn_is_ldp(insn) ||
aarch64_insn_is_ldp_pre(insn) ||
aarch64_insn_is_ldp_post(insn);
}
+static __always_inline bool aarch64_insn_uses_literal(u32 insn)
+{
+ /* ldr/ldrsw (literal), prfm */
+
+ return aarch64_insn_is_ldr_lit(insn) ||
+ aarch64_insn_is_ldrsw_lit(insn) ||
+ aarch64_insn_is_adr_adrp(insn) ||
+ aarch64_insn_is_prfm_lit(insn);
+}
+
enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
-bool aarch64_insn_uses_literal(u32 insn);
-bool aarch64_insn_is_branch(u32 insn);
u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn);
u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
u32 insn, u64 imm);
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index ac0df03fffcd..67c1be804cc3 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -44,67 +44,6 @@ enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
}
-bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
-{
- if (!aarch64_insn_is_hint(insn))
- return false;
-
- switch (insn & 0xFE0) {
- case AARCH64_INSN_HINT_XPACLRI:
- case AARCH64_INSN_HINT_PACIA_1716:
- case AARCH64_INSN_HINT_PACIB_1716:
- case AARCH64_INSN_HINT_PACIAZ:
- case AARCH64_INSN_HINT_PACIASP:
- case AARCH64_INSN_HINT_PACIBZ:
- case AARCH64_INSN_HINT_PACIBSP:
- case AARCH64_INSN_HINT_BTI:
- case AARCH64_INSN_HINT_BTIC:
- case AARCH64_INSN_HINT_BTIJ:
- case AARCH64_INSN_HINT_BTIJC:
- case AARCH64_INSN_HINT_NOP:
- return true;
- default:
- return false;
- }
-}
-
-bool aarch64_insn_is_branch_imm(u32 insn)
-{
- return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) ||
- aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) ||
- aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
- aarch64_insn_is_bcond(insn));
-}
-
-bool __kprobes aarch64_insn_uses_literal(u32 insn)
-{
- /* ldr/ldrsw (literal), prfm */
-
- return aarch64_insn_is_ldr_lit(insn) ||
- aarch64_insn_is_ldrsw_lit(insn) ||
- aarch64_insn_is_adr_adrp(insn) ||
- aarch64_insn_is_prfm_lit(insn);
-}
-
-bool __kprobes aarch64_insn_is_branch(u32 insn)
-{
- /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
-
- return aarch64_insn_is_b(insn) ||
- aarch64_insn_is_bl(insn) ||
- aarch64_insn_is_cbz(insn) ||
- aarch64_insn_is_cbnz(insn) ||
- aarch64_insn_is_tbz(insn) ||
- aarch64_insn_is_tbnz(insn) ||
- aarch64_insn_is_ret(insn) ||
- aarch64_insn_is_ret_auth(insn) ||
- aarch64_insn_is_br(insn) ||
- aarch64_insn_is_br_auth(insn) ||
- aarch64_insn_is_blr(insn) ||
- aarch64_insn_is_blr_auth(insn) ||
- aarch64_insn_is_bcond(insn);
-}
-
static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
u32 *maskp, int *shiftp)
{
From 4488f90c8609e5c420531d604dd19cdfee4ec0e0 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 13:59:27 +0000
Subject: [PATCH 039/151] arm64: insn: simplify insn group identification
The only code which needs to check for an entire instruction group is
the aarch64_insn_is_steppable() helper function used by kprobes, which
must not be instrumented, and only needs to check for the "Branch,
exception generation and system instructions" class.
Currently we have an out-of-line helper in insn.c which must be marked
as __kprobes, which indexes a table with some bits extracted from the
instruction. In aarch64_insn_is_steppable() we then need to compare the
result with an expected enum value.
It would be simpler to have a predicate for this, as with the other
aarch64_insn_is_*() helpers, which would be always inlined to prevent
inadvertent instrumentation, and would permit better code generation.
This patch adds a predicate function for this instruction group using
the existing __AARCH64_INSN_FUNCS() helpers, and removes the existing
out-of-line helper. As the only class we currently care about is the
branch+exception+sys class, I have only added helpers for this, and left
the other classes unimplemented for now.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Joey Gouly <joey.gouly@arm.com>
Link: https://lore.kernel.org/r/20221114135928.3000571-4-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/insn.h | 42 +++++++++++---------------
arch/arm64/kernel/probes/decode-insn.c | 2 +-
arch/arm64/lib/insn.c | 24 ---------------
3 files changed, 18 insertions(+), 50 deletions(-)
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index de5bc71b4b1d..876cacad103e 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -13,31 +13,6 @@
#include <asm/insn-def.h>
#ifndef __ASSEMBLY__
-/*
- * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
- * Section C3.1 "A64 instruction index by encoding":
- * AArch64 main encoding table
- * Bit position
- * 28 27 26 25 Encoding Group
- * 0 0 - - Unallocated
- * 1 0 0 - Data processing, immediate
- * 1 0 1 - Branch, exception generation and system instructions
- * - 1 - 0 Loads and stores
- * - 1 0 1 Data processing - register
- * 0 1 1 1 Data processing - SIMD and floating point
- * 1 1 1 1 Data processing - SIMD and floating point
- * "-" means "don't care"
- */
-enum aarch64_insn_encoding_class {
- AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
- AARCH64_INSN_CLS_SVE, /* SVE instructions */
- AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
- AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
- AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
- AARCH64_INSN_CLS_LDST, /* Loads and stores */
- AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and
- * system instructions */
-};
enum aarch64_insn_hint_cr_op {
AARCH64_INSN_HINT_NOP = 0x0 << 5,
@@ -326,6 +301,23 @@ static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \
return (val); \
}
+/*
+ * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
+ * Section C3.1 "A64 instruction index by encoding":
+ * AArch64 main encoding table
+ * Bit position
+ * 28 27 26 25 Encoding Group
+ * 0 0 - - Unallocated
+ * 1 0 0 - Data processing, immediate
+ * 1 0 1 - Branch, exception generation and system instructions
+ * - 1 - 0 Loads and stores
+ * - 1 0 1 Data processing - register
+ * 0 1 1 1 Data processing - SIMD and floating point
+ * 1 1 1 1 Data processing - SIMD and floating point
+ * "-" means "don't care"
+ */
+__AARCH64_INSN_FUNCS(class_branch_sys, 0x1c000000, 0x14000000)
+
__AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000)
__AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000)
__AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000)
diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c
index 104101f633b1..968d5fffe233 100644
--- a/arch/arm64/kernel/probes/decode-insn.c
+++ b/arch/arm64/kernel/probes/decode-insn.c
@@ -24,7 +24,7 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn)
* currently safe. Lastly, MSR instructions can do any number of nasty
* things we can't handle during single-stepping.
*/
- if (aarch64_get_insn_class(insn) == AARCH64_INSN_CLS_BR_SYS) {
+ if (aarch64_insn_is_class_branch_sys(insn)) {
if (aarch64_insn_is_branch(insn) ||
aarch64_insn_is_msr_imm(insn) ||
aarch64_insn_is_msr_reg(insn) ||
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index 67c1be804cc3..99194464d675 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -20,30 +20,6 @@
#define AARCH64_INSN_N_BIT BIT(22)
#define AARCH64_INSN_LSL_12 BIT(22)
-static const int aarch64_insn_encoding_class[] = {
- AARCH64_INSN_CLS_UNKNOWN,
- AARCH64_INSN_CLS_UNKNOWN,
- AARCH64_INSN_CLS_SVE,
- AARCH64_INSN_CLS_UNKNOWN,
- AARCH64_INSN_CLS_LDST,
- AARCH64_INSN_CLS_DP_REG,
- AARCH64_INSN_CLS_LDST,
- AARCH64_INSN_CLS_DP_FPSIMD,
- AARCH64_INSN_CLS_DP_IMM,
- AARCH64_INSN_CLS_DP_IMM,
- AARCH64_INSN_CLS_BR_SYS,
- AARCH64_INSN_CLS_BR_SYS,
- AARCH64_INSN_CLS_LDST,
- AARCH64_INSN_CLS_DP_REG,
- AARCH64_INSN_CLS_LDST,
- AARCH64_INSN_CLS_DP_FPSIMD,
-};
-
-enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
-{
- return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
-}
-
static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
u32 *maskp, int *shiftp)
{
From f750255fdad33e8ac46eadf225d6764148e4642e Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 13:59:28 +0000
Subject: [PATCH 040/151] arm64: insn: always inline hint generation
All users of aarch64_insn_gen_hint() (e.g. aarch64_insn_gen_nop()) pass
a constant argument and generate a constant value. Some of those users
are noinstr code (e.g. for alternatives patching).
For noinstr code it is necessary to either inline these functions or to
ensure the out-of-line versions are noinstr.
Since in all cases these are generating a constant, make them
__always_inline.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Joey Gouly <joey.gouly@arm.com>
Link: https://lore.kernel.org/r/20221114135928.3000571-5-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/insn.h | 14 ++++++++++++--
arch/arm64/lib/insn.c | 10 ----------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 876cacad103e..aaf1f52fbf3e 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -552,8 +552,18 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_branch_type type);
u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_condition cond);
-u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op);
-u32 aarch64_insn_gen_nop(void);
+
+static __always_inline u32
+aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
+{
+ return aarch64_insn_get_hint_value() | op;
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+ return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}
+
u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
enum aarch64_insn_branch_type type);
u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index 99194464d675..924934cb85ee 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -350,16 +350,6 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
offset >> 2);
}
-u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
-{
- return aarch64_insn_get_hint_value() | op;
-}
-
-u32 __kprobes aarch64_insn_gen_nop(void)
-{
- return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
-}
-
u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
enum aarch64_insn_branch_type type)
{
From 39522031798dbb53b169b95d4ab25b53301d4eaf Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Fri, 11 Nov 2022 17:43:23 -0600
Subject: [PATCH 041/151] ACPI: APMT: Fix kerneldoc and indentation
Add missing kerneldoc and fix alignment on one of the arguments of
apmt_add_platform_device function.
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Link: https://lore.kernel.org/r/20221111234323.16182-1-bwicaksono@nvidia.com
[will: Fixed up additional indentation issue]
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/acpi/arm64/apmt.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c
index f55167ca51e7..8cab69fa5d59 100644
--- a/drivers/acpi/arm64/apmt.c
+++ b/drivers/acpi/arm64/apmt.c
@@ -24,7 +24,7 @@
static struct acpi_table_header *apmt_table;
static int __init apmt_init_resources(struct resource *res,
- struct acpi_apmt_node *node)
+ struct acpi_apmt_node *node)
{
int irq, trigger;
int num_res = 0;
@@ -66,11 +66,12 @@ static int __init apmt_init_resources(struct resource *res,
/**
* apmt_add_platform_device() - Allocate a platform device for APMT node
* @node: Pointer to device ACPI APMT node
+ * @fwnode: fwnode associated with the APMT node
*
* Returns: 0 on success, <0 failure
*/
static int __init apmt_add_platform_device(struct acpi_apmt_node *node,
- struct fwnode_handle *fwnode)
+ struct fwnode_handle *fwnode)
{
struct platform_device *pdev;
int ret, count;
From 0bb1fbffc631064db567ccaeb9ed6b6df6342b66 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Mon, 14 Nov 2022 10:44:11 +0000
Subject: [PATCH 042/151] arm64: mm: kfence: only handle translation faults
Alexander noted that KFENCE only expects to handle faults from invalid page
table entries (i.e. translation faults), but arm64's fault handling logic will
call kfence_handle_page_fault() for other types of faults, including alignment
faults caused by unaligned atomics. This has the unfortunate property of
causing those other faults to be reported as "KFENCE: use-after-free",
which is misleading and hinders debugging.
Fix this by only forwarding unhandled translation faults to the KFENCE
code, similar to what x86 does already.
Alexander has verified that this passes all the tests in the KFENCE test
suite and avoids bogus reports on misaligned atomics.
Link: https://lore.kernel.org/all/20221102081620.1465154-1-zhongbaisong@huawei.com/
Fixes: 840b23986344 ("arm64, kfence: enable KFENCE for ARM64")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Alexander Potapenko <glider@google.com>
Tested-by: Alexander Potapenko <glider@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Marco Elver <elver@google.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221114104411.2853040-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/mm/fault.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 3e9cf9826417..3eb2825d08cf 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -354,6 +354,11 @@ static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
return false;
}
+static bool is_translation_fault(unsigned long esr)
+{
+ return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_FAULT;
+}
+
static void __do_kernel_fault(unsigned long addr, unsigned long esr,
struct pt_regs *regs)
{
@@ -386,7 +391,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr,
} else if (addr < PAGE_SIZE) {
msg = "NULL pointer dereference";
} else {
- if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
+ if (is_translation_fault(esr) &&
+ kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
return;
msg = "paging request";
From facafab7611f7b872c6b9eeaff53461ef11f482e Mon Sep 17 00:00:00 2001
From: Yuan Can <yuancan@huawei.com>
Date: Tue, 15 Nov 2022 07:02:06 +0000
Subject: [PATCH 043/151] perf: arm_dsu: Fix hotplug callback leak in
dsu_pmu_init()
dsu_pmu_init() won't remove the callback added by cpuhp_setup_state_multi()
when platform_driver_register() failed. Remove the callback by
cpuhp_remove_multi_state() in fail path.
Similar to the handling of arm_ccn_init() in commit 26242b330093 ("bus:
arm-ccn: Prevent hotplug callback leak")
Fixes: 7520fa99246d ("perf: ARM DynamIQ Shared Unit PMU support")
Signed-off-by: Yuan Can <yuancan@huawei.com>
Acked-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Link: https://lore.kernel.org/r/20221115070207.32634-2-yuancan@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_dsu_pmu.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c
index 4a15c86f45ef..fe2abb412c00 100644
--- a/drivers/perf/arm_dsu_pmu.c
+++ b/drivers/perf/arm_dsu_pmu.c
@@ -858,7 +858,11 @@ static int __init dsu_pmu_init(void)
if (ret < 0)
return ret;
dsu_pmu_cpuhp_state = ret;
- return platform_driver_register(&dsu_pmu_driver);
+ ret = platform_driver_register(&dsu_pmu_driver);
+ if (ret)
+ cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
+
+ return ret;
}
static void __exit dsu_pmu_exit(void)
From 973ae93d80d9d262f695eb485a1902b74c4b9098 Mon Sep 17 00:00:00 2001
From: Yuan Can <yuancan@huawei.com>
Date: Tue, 15 Nov 2022 07:02:07 +0000
Subject: [PATCH 044/151] drivers: perf: marvell_cn10k: Fix hotplug callback
leak in tad_pmu_init()
tad_pmu_init() won't remove the callback added by cpuhp_setup_state_multi()
when platform_driver_register() failed. Remove the callback by
cpuhp_remove_multi_state() in fail path.
Similar to the handling of arm_ccn_init() in commit 26242b330093 ("bus:
arm-ccn: Prevent hotplug callback leak")
Fixes: 036a7584bede ("drivers: perf: Add LLC-TAD perf counter support")
Signed-off-by: Yuan Can <yuancan@huawei.com>
Link: https://lore.kernel.org/r/20221115070207.32634-3-yuancan@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/marvell_cn10k_tad_pmu.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/perf/marvell_cn10k_tad_pmu.c b/drivers/perf/marvell_cn10k_tad_pmu.c
index 69c3050a4348..a1166afb3702 100644
--- a/drivers/perf/marvell_cn10k_tad_pmu.c
+++ b/drivers/perf/marvell_cn10k_tad_pmu.c
@@ -408,7 +408,11 @@ static int __init tad_pmu_init(void)
if (ret < 0)
return ret;
tad_pmu_cpuhp_state = ret;
- return platform_driver_register(&tad_pmu_driver);
+ ret = platform_driver_register(&tad_pmu_driver);
+ if (ret)
+ cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
+
+ return ret;
}
static void __exit tad_pmu_exit(void)
From b3a0c010e900a9f89dcd99f10bd8f7538d21b0a9 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:15 +0100
Subject: [PATCH 045/151] arm64: allow kprobes on EL0 handlers
Currently do_sysinstr() and do_cp15instr() are marked with
NOKPROBE_SYMBOL(). However, these are only called for exceptions taken
from EL0, and there is no risk of recursion in kprobes, so this is not
necessary.
Remove the NOKPROBE_SYMBOL() annotation, and rename the two functions to
more clearly indicate that these are solely for exceptions taken from
EL0, better matching the names used by the lower level entry points in
entry-common.c.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-2-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/exception.h | 4 ++--
arch/arm64/kernel/entry-common.c | 4 ++--
arch/arm64/kernel/traps.c | 6 ++----
3 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 19713d0f013b..bc0dd002106d 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -67,10 +67,10 @@ void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs);
void do_sve_acc(unsigned long esr, struct pt_regs *regs);
void do_sme_acc(unsigned long esr, struct pt_regs *regs);
void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs);
-void do_sysinstr(unsigned long esr, struct pt_regs *regs);
+void do_el0_sys(unsigned long esr, struct pt_regs *regs);
void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs);
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
-void do_cp15instr(unsigned long esr, struct pt_regs *regs);
+void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 27369fa1c032..da4bb2bd2420 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -570,7 +570,7 @@ static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_sysinstr(esr, regs);
+ do_el0_sys(esr, regs);
exit_to_user_mode(regs);
}
@@ -762,7 +762,7 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_cp15instr(esr, regs);
+ do_el0_cp15(esr, regs);
exit_to_user_mode(regs);
}
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 23d281ed7621..ae184d661067 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -748,7 +748,7 @@ static const struct sys64_hook cp15_64_hooks[] = {
{},
};
-void do_cp15instr(unsigned long esr, struct pt_regs *regs)
+void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook, *hook_base;
@@ -786,10 +786,9 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
*/
do_undefinstr(regs, esr);
}
-NOKPROBE_SYMBOL(do_cp15instr);
#endif
-void do_sysinstr(unsigned long esr, struct pt_regs *regs)
+void do_el0_sys(unsigned long esr, struct pt_regs *regs)
{
const struct sys64_hook *hook;
@@ -806,7 +805,6 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs)
*/
do_undefinstr(regs, esr);
}
-NOKPROBE_SYMBOL(do_sysinstr);
static const char *esr_class_str[] = {
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
From 61d64a376ea80f9097e7ea599bcd68671b836dc6 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:16 +0100
Subject: [PATCH 046/151] arm64: split EL0/EL1 UNDEF handlers
In general, exceptions taken from EL1 need to be handled separately from
exceptions taken from EL0, as the logic to handle the two cases can be
significantly divergent, and exceptions taken from EL1 typically have
more stringent requirements on locking and instrumentation.
Subsequent patches will rework the way EL1 UNDEFs are handled in order
to address longstanding soundness issues with instrumentation and RCU.
In preparation for that rework, this patch splits the existing
do_undefinstr() handler into separate do_el0_undef() and do_el1_undef()
handlers.
Prior to this patch, do_undefinstr() was marked with NOKPROBE_SYMBOL(),
preventing instrumentation via kprobes. However, do_undefinstr() invokes
other code which can be instrumented, and:
* For UNDEFINED exceptions taken from EL0, there is no risk of recursion
within kprobes. Therefore it is safe for do_el0_undef to be
instrumented with kprobes, and it does not need to be marked with
NOKPROBE_SYMBOL().
* For UNDEFINED exceptions taken from EL1, either:
(a) The exception is has been taken when manipulating SSBS; these cases
are limited and do not occur within code that can be invoked
recursively via kprobes. Hence, in these cases instrumentation
with kprobes is benign.
(b) The exception has been taken for an unknown reason, as other than
manipulating SSBS we do not expect to take UNDEFINED exceptions
from EL1. Any handling of these exception is best-effort.
... and in either case, marking do_el1_undef() with NOKPROBE_SYMBOL()
isn't sufficient to prevent recursion via kprobes as functions it
calls (including die()) are instrumentable via kprobes.
Hence, it's not worthwhile to mark do_el1_undef() with
NOKPROBE_SYMBOL(). The same applies to do_el1_bti() and do_el1_fpac(),
so their NOKPROBE_SYMBOL() annotations are also removed.
Aside from the new instrumentability, there should be no functional
change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-3-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/exception.h | 3 ++-
arch/arm64/kernel/entry-common.c | 4 ++--
arch/arm64/kernel/traps.c | 22 ++++++++++++----------
3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index bc0dd002106d..92963f98afec 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -58,7 +58,8 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs,
asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs);
-void do_undefinstr(struct pt_regs *regs, unsigned long esr);
+void do_el0_undef(struct pt_regs *regs, unsigned long esr);
+void do_el1_undef(struct pt_regs *regs, unsigned long esr);
void do_el0_bti(struct pt_regs *regs);
void do_el1_bti(struct pt_regs *regs, unsigned long esr);
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index da4bb2bd2420..d0ac4e4dfff8 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -384,7 +384,7 @@ static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_kernel_mode(regs);
local_daif_inherit(regs);
- do_undefinstr(regs, esr);
+ do_el1_undef(regs, esr);
local_daif_mask();
exit_to_kernel_mode(regs);
}
@@ -599,7 +599,7 @@ static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
{
enter_from_user_mode(regs);
local_daif_restore(DAIF_PROCCTX);
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
exit_to_user_mode(regs);
}
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index ae184d661067..678720103388 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -486,7 +486,7 @@ void arm64_notify_segfault(unsigned long addr)
force_signal_inject(SIGSEGV, code, addr, 0);
}
-void do_undefinstr(struct pt_regs *regs, unsigned long esr)
+void do_el0_undef(struct pt_regs *regs, unsigned long esr)
{
/* check for AArch32 breakpoint instructions */
if (!aarch32_break_handler(regs))
@@ -495,12 +495,16 @@ void do_undefinstr(struct pt_regs *regs, unsigned long esr)
if (call_undef_hook(regs) == 0)
return;
- if (!user_mode(regs))
- die("Oops - Undefined instruction", regs, esr);
-
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
-NOKPROBE_SYMBOL(do_undefinstr);
+
+void do_el1_undef(struct pt_regs *regs, unsigned long esr)
+{
+ if (call_undef_hook(regs) == 0)
+ return;
+
+ die("Oops - Undefined instruction", regs, esr);
+}
void do_el0_bti(struct pt_regs *regs)
{
@@ -511,7 +515,6 @@ void do_el1_bti(struct pt_regs *regs, unsigned long esr)
{
die("Oops - BTI", regs, esr);
}
-NOKPROBE_SYMBOL(do_el1_bti);
void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
{
@@ -526,7 +529,6 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
*/
die("Oops - FPAC", regs, esr);
}
-NOKPROBE_SYMBOL(do_el1_fpac)
#define __user_cache_maint(insn, address, res) \
if (address >= TASK_SIZE_MAX) { \
@@ -769,7 +771,7 @@ void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
hook_base = cp15_64_hooks;
break;
default:
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
return;
}
@@ -784,7 +786,7 @@ void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
* EL0. Fall back to our usual undefined instruction handler
* so that we handle these consistently.
*/
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
}
#endif
@@ -803,7 +805,7 @@ void do_el0_sys(unsigned long esr, struct pt_regs *regs)
* back to our usual undefined instruction handler so that we handle
* these consistently.
*/
- do_undefinstr(regs, esr);
+ do_el0_undef(regs, esr);
}
static const char *esr_class_str[] = {
From bff8f413c71ffc3cb679dbd9a5632b33af563f9f Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:17 +0100
Subject: [PATCH 047/151] arm64: factor out EL1 SSBS emulation hook
Currently call_undef_hook() is used to handle UNDEFINED exceptions from
EL0 and EL1. As support for deprecated instructions may be enabled
independently, the handlers for individual instructions are organised as
a linked list of struct undef_hook which can be manipulated dynamically.
As this can be manipulated dynamically, the list is protected with a
raw_spinlock which must be acquired when handling UNDEFINED exceptions
or when manipulating the list of handlers.
This locking is unfortunate as it serialises handling of UNDEFINED
exceptions, and requires RCU to be enabled for lockdep, requiring the
use of RCU_NONIDLE() in resume path of cpu_suspend() since commit:
a2c42bbabbe260b7 ("arm64: spectre: Prevent lockdep splat on v4 mitigation enable path")
The list of UNDEFINED handlers largely consist of handlers for
exceptions taken from EL0, and the only handler for exceptions taken
from EL1 handles `MSR SSBS, #imm` on CPUs which feature PSTATE.SSBS but
lack the corresponding MSR (Immediate) instruction. Other than this we
never expect to take an UNDEFINED exception from EL1 in normal
operation.
This patch reworks do_el0_undef() to invoke the EL1 SSBS handler
directly, relegating call_undef_hook() to only handle EL0 UNDEFs. This
removes redundant work to iterate the list for EL1 UNDEFs, and removes
the need for locking, permitting EL1 UNDEFs to be handled in parallel
without contention.
The RCU_NONIDLE() call in cpu_suspend() will be removed in a subsequent
patch, as there are other potential issues with the use of
instrumentable code and RCU in the CPU suspend code.
I've tested this by forcing the detection of SSBS on a CPU that doesn't
have it, and verifying that the try_emulate_el1_ssbs() callback is
invoked.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-4-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/spectre.h | 2 ++
arch/arm64/kernel/proton-pack.c | 26 +++++++-------------------
arch/arm64/kernel/traps.c | 15 ++++++++-------
3 files changed, 17 insertions(+), 26 deletions(-)
diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h
index aa3d3607d5c8..db7b371b367c 100644
--- a/arch/arm64/include/asm/spectre.h
+++ b/arch/arm64/include/asm/spectre.h
@@ -26,6 +26,7 @@ enum mitigation_state {
SPECTRE_VULNERABLE,
};
+struct pt_regs;
struct task_struct;
/*
@@ -98,5 +99,6 @@ enum mitigation_state arm64_get_spectre_bhb_state(void);
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
u8 spectre_bhb_loop_affected(int scope);
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr);
#endif /* __ASSEMBLY__ */
#endif /* __ASM_SPECTRE_H */
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index bfce41c2a53b..fca9cc6f5581 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -521,10 +521,13 @@ bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
return state != SPECTRE_UNAFFECTED;
}
-static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
{
- if (user_mode(regs))
- return 1;
+ const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
+ const u32 instr_val = 0xd500401f | PSTATE_SSBS;
+
+ if ((instr & instr_mask) != instr_val)
+ return false;
if (instr & BIT(PSTATE_Imm_shift))
regs->pstate |= PSR_SSBS_BIT;
@@ -532,19 +535,11 @@ static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
regs->pstate &= ~PSR_SSBS_BIT;
arm64_skip_faulting_instruction(regs, 4);
- return 0;
+ return true;
}
-static struct undef_hook ssbs_emulation_hook = {
- .instr_mask = ~(1U << PSTATE_Imm_shift),
- .instr_val = 0xd500401f | PSTATE_SSBS,
- .fn = ssbs_emulation_handler,
-};
-
static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
{
- static bool undef_hook_registered = false;
- static DEFINE_RAW_SPINLOCK(hook_lock);
enum mitigation_state state;
/*
@@ -555,13 +550,6 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
return state;
- raw_spin_lock(&hook_lock);
- if (!undef_hook_registered) {
- register_undef_hook(&ssbs_emulation_hook);
- undef_hook_registered = true;
- }
- raw_spin_unlock(&hook_lock);
-
if (spectre_v4_mitigations_off()) {
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
set_pstate_ssbs(1);
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 678720103388..ccd7d773e5cd 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -402,12 +402,7 @@ static int call_undef_hook(struct pt_regs *regs)
int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
unsigned long pc = instruction_pointer(regs);
- if (!user_mode(regs)) {
- __le32 instr_le;
- if (get_kernel_nofault(instr_le, (__le32 *)pc))
- goto exit;
- instr = le32_to_cpu(instr_le);
- } else if (compat_thumb_mode(regs)) {
+ if (compat_thumb_mode(regs)) {
/* 16-bit Thumb instruction */
__le16 instr_le;
if (get_user(instr_le, (__le16 __user *)pc))
@@ -500,9 +495,15 @@ void do_el0_undef(struct pt_regs *regs, unsigned long esr)
void do_el1_undef(struct pt_regs *regs, unsigned long esr)
{
- if (call_undef_hook(regs) == 0)
+ u32 insn;
+
+ if (aarch64_insn_read((void *)regs->pc, &insn))
+ goto out_err;
+
+ if (try_emulate_el1_ssbs(regs, insn))
return;
+out_err:
die("Oops - Undefined instruction", regs, esr);
}
From dbfbd87efa79575491af0ba1a87bf567eaea6cae Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:18 +0100
Subject: [PATCH 048/151] arm64: factor insn read out of call_undef_hook()
Subsequent patches will rework EL0 UNDEF handling, removing the need for
struct undef_hook and call_undef_hook. In preparation for those changes,
this patch factors the logic for reading user instructions out of
call_undef_hook() and into a new user_insn_read() helper, matching the
style of the existing aarch64_insn_read() helper used for reading kernel
instructions.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-5-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/traps.c | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index ccd7d773e5cd..4d51afd010e1 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -394,25 +394,22 @@ void unregister_undef_hook(struct undef_hook *hook)
raw_spin_unlock_irqrestore(&undef_lock, flags);
}
-static int call_undef_hook(struct pt_regs *regs)
+static int user_insn_read(struct pt_regs *regs, u32 *insnp)
{
- struct undef_hook *hook;
- unsigned long flags;
u32 instr;
- int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
unsigned long pc = instruction_pointer(regs);
if (compat_thumb_mode(regs)) {
/* 16-bit Thumb instruction */
__le16 instr_le;
if (get_user(instr_le, (__le16 __user *)pc))
- goto exit;
+ return -EFAULT;
instr = le16_to_cpu(instr_le);
if (aarch32_insn_is_wide(instr)) {
u32 instr2;
if (get_user(instr_le, (__le16 __user *)(pc + 2)))
- goto exit;
+ return -EFAULT;
instr2 = le16_to_cpu(instr_le);
instr = (instr << 16) | instr2;
}
@@ -420,10 +417,20 @@ static int call_undef_hook(struct pt_regs *regs)
/* 32-bit ARM instruction */
__le32 instr_le;
if (get_user(instr_le, (__le32 __user *)pc))
- goto exit;
+ return -EFAULT;
instr = le32_to_cpu(instr_le);
}
+ *insnp = instr;
+ return 0;
+}
+
+static int call_undef_hook(struct pt_regs *regs, u32 instr)
+{
+ struct undef_hook *hook;
+ unsigned long flags;
+ int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
+
raw_spin_lock_irqsave(&undef_lock, flags);
list_for_each_entry(hook, &undef_hook, node)
if ((instr & hook->instr_mask) == hook->instr_val &&
@@ -431,7 +438,7 @@ static int call_undef_hook(struct pt_regs *regs)
fn = hook->fn;
raw_spin_unlock_irqrestore(&undef_lock, flags);
-exit:
+
return fn ? fn(regs, instr) : 1;
}
@@ -483,13 +490,19 @@ void arm64_notify_segfault(unsigned long addr)
void do_el0_undef(struct pt_regs *regs, unsigned long esr)
{
+ u32 insn;
+
/* check for AArch32 breakpoint instructions */
if (!aarch32_break_handler(regs))
return;
- if (call_undef_hook(regs) == 0)
+ if (user_insn_read(regs, &insn))
+ goto out_err;
+
+ if (call_undef_hook(regs, insn) == 0)
return;
+out_err:
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
From f5962add74b61f8ae31c6311f75ca35d7e1d2d8f Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:19 +0100
Subject: [PATCH 049/151] arm64: rework EL0 MRS emulation
On CPUs without FEAT_IDST, ID register emulation is slower than it needs
to be, as all threads contend for the same lock to perform the
emulation. This patch reworks the emulation to avoid this unnecessary
contention.
On CPUs with FEAT_IDST (which is mandatory from ARMv8.4 onwards), EL0
accesses to ID registers result in a SYS trap, and emulation of these is
handled with a sys64_hook. These hooks are statically allocated, and no
locking is required to iterate through the hooks and perform the
emulation, allowing emulation to occur in parallel with no contention.
On CPUs without FEAT_IDST, EL0 accesses to ID registers result in an
UNDEFINED exception, and emulation of these accesses is handled with an
undef_hook. When an EL0 MRS instruction is trapped to EL1, the kernel
finds the relevant handler by iterating through all of the undef_hooks,
requiring undef_lock to be held during this lookup.
This locking is only required to safely traverse the list of undef_hooks
(as it can be concurrently modified), and the actual emulation of the
MRS does not require any mutual exclusion. This locking is an
unfortunate bottleneck, especially given that MRS emulation is enabled
unconditionally and is never disabled.
This patch reworks the non-FEAT_IDST MRS emulation logic so that it can
be invoked directly from do_el0_undef(). This removes the bottleneck,
allowing MRS traps to be handled entirely in parallel, and is a stepping
stone to making all of the undef_hooks lock-free.
I've tested this in a 64-vCPU VM on a 64-CPU ThunderX2 host, with a
benchmark which spawns a number of threads which each try to read
ID_AA64ISAR0_EL1 1000000 times. This is vastly more contention than will
ever be seen in realistic usage, but clearly demonstrates the removal of
the bottleneck:
| Threads || Time (seconds) |
| || Before || After |
| || Real | System || Real | System |
|---------++--------+---------++--------+---------|
| 1 || 0.29 | 0.20 || 0.24 | 0.12 |
| 2 || 0.35 | 0.51 || 0.23 | 0.27 |
| 4 || 1.08 | 3.87 || 0.24 | 0.56 |
| 8 || 4.31 | 33.60 || 0.24 | 1.11 |
| 16 || 9.47 | 149.39 || 0.23 | 2.15 |
| 32 || 19.07 | 605.27 || 0.24 | 4.38 |
| 64 || 65.40 | 3609.09 || 0.33 | 11.27 |
Aside from the speedup, there should be no functional change as a result
of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-6-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/cpufeature.h | 3 ++-
arch/arm64/kernel/cpufeature.c | 23 +++++------------------
arch/arm64/kernel/traps.c | 3 +++
3 files changed, 10 insertions(+), 19 deletions(-)
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index f73f11b55042..03d1c9d7af82 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -832,7 +832,8 @@ static inline bool system_supports_tlb_range(void)
cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
}
-extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
+int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
+bool try_emulate_mrs(struct pt_regs *regs, u32 isn);
static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
{
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b3f37e2209ad..9592e75b07cd 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -3435,35 +3435,22 @@ int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
return rc;
}
-static int emulate_mrs(struct pt_regs *regs, u32 insn)
+bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
{
u32 sys_reg, rt;
+ if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn))
+ return false;
+
/*
* sys_reg values are defined as used in mrs/msr instruction.
* shift the imm value to get the encoding.
*/
sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
- return do_emulate_mrs(regs, sys_reg, rt);
+ return do_emulate_mrs(regs, sys_reg, rt) == 0;
}
-static struct undef_hook mrs_hook = {
- .instr_mask = 0xffff0000,
- .instr_val = 0xd5380000,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_MODE_EL0t,
- .fn = emulate_mrs,
-};
-
-static int __init enable_mrs_emulation(void)
-{
- register_undef_hook(&mrs_hook);
- return 0;
-}
-
-core_initcall(enable_mrs_emulation);
-
enum mitigation_state arm64_get_meltdown_state(void)
{
if (__meltdown_safe)
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 4d51afd010e1..96eaf1aaec12 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -499,6 +499,9 @@ void do_el0_undef(struct pt_regs *regs, unsigned long esr)
if (user_insn_read(regs, &insn))
goto out_err;
+ if (try_emulate_mrs(regs, insn))
+ return;
+
if (call_undef_hook(regs, insn) == 0)
return;
From b4453cc8a7ebbd45436a8cd3ffeaa069ceac146f Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:20 +0100
Subject: [PATCH 050/151] arm64: armv8_deprecated: fold ops into insn_emulation
The code for emulating deprecated instructions has two related
structures: struct insn_emulation_ops and struct insn_emulation, where
each struct insn_emulation_ops is associated 1-1 with a struct
insn_emulation.
It would be simpler to combine the two into a single structure, removing
the need for (unconditional) dynamic allocation at boot time, and
simplifying some runtime pointer chasing.
This patch merges the two structures together.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-7-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/armv8_deprecated.c | 76 ++++++++++++----------------
1 file changed, 33 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index fb0e7c7b2e20..2a75e20a3913 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -41,16 +41,12 @@ enum legacy_insn_status {
INSN_OBSOLETE,
};
-struct insn_emulation_ops {
- const char *name;
- enum legacy_insn_status status;
- struct undef_hook *hooks;
- int (*set_hw_mode)(bool enable);
-};
-
struct insn_emulation {
- struct list_head node;
- struct insn_emulation_ops *ops;
+ const char *name;
+ struct list_head node;
+ enum legacy_insn_status status;
+ struct undef_hook *hooks;
+ int (*set_hw_mode)(bool enable);
int current_mode;
int min;
int max;
@@ -61,48 +57,48 @@ static int nr_insn_emulated __initdata;
static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
static DEFINE_MUTEX(insn_emulation_mutex);
-static void register_emulation_hooks(struct insn_emulation_ops *ops)
+static void register_emulation_hooks(struct insn_emulation *insn)
{
struct undef_hook *hook;
- BUG_ON(!ops->hooks);
+ BUG_ON(!insn->hooks);
- for (hook = ops->hooks; hook->instr_mask; hook++)
+ for (hook = insn->hooks; hook->instr_mask; hook++)
register_undef_hook(hook);
- pr_notice("Registered %s emulation handler\n", ops->name);
+ pr_notice("Registered %s emulation handler\n", insn->name);
}
-static void remove_emulation_hooks(struct insn_emulation_ops *ops)
+static void remove_emulation_hooks(struct insn_emulation *insn)
{
struct undef_hook *hook;
- BUG_ON(!ops->hooks);
+ BUG_ON(!insn->hooks);
- for (hook = ops->hooks; hook->instr_mask; hook++)
+ for (hook = insn->hooks; hook->instr_mask; hook++)
unregister_undef_hook(hook);
- pr_notice("Removed %s emulation handler\n", ops->name);
+ pr_notice("Removed %s emulation handler\n", insn->name);
}
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->ops->set_hw_mode)
- insn->ops->set_hw_mode(true);
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(true);
}
static void disable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->ops->set_hw_mode)
- insn->ops->set_hw_mode(false);
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(false);
}
/* Run set_hw_mode(mode) on all active CPUs */
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
{
- if (!insn->ops->set_hw_mode)
+ if (!insn->set_hw_mode)
return -EINVAL;
if (enable)
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
@@ -126,9 +122,9 @@ static int run_all_insn_set_hw_mode(unsigned int cpu)
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_for_each_entry(insn, &insn_emulation, node) {
bool enable = (insn->current_mode == INSN_HW);
- if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
+ if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
pr_warn("CPU[%u] cannot support the emulation of %s",
- cpu, insn->ops->name);
+ cpu, insn->name);
rc = -EINVAL;
}
}
@@ -145,11 +141,11 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
case INSN_UNDEF: /* Nothing to be done */
break;
case INSN_EMULATE:
- remove_emulation_hooks(insn->ops);
+ remove_emulation_hooks(insn);
break;
case INSN_HW:
if (!run_all_cpu_set_hw_mode(insn, false))
- pr_notice("Disabled %s support\n", insn->ops->name);
+ pr_notice("Disabled %s support\n", insn->name);
break;
}
@@ -157,31 +153,25 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
case INSN_UNDEF:
break;
case INSN_EMULATE:
- register_emulation_hooks(insn->ops);
+ register_emulation_hooks(insn);
break;
case INSN_HW:
ret = run_all_cpu_set_hw_mode(insn, true);
if (!ret)
- pr_notice("Enabled %s support\n", insn->ops->name);
+ pr_notice("Enabled %s support\n", insn->name);
break;
}
return ret;
}
-static void __init register_insn_emulation(struct insn_emulation_ops *ops)
+static void __init register_insn_emulation(struct insn_emulation *insn)
{
unsigned long flags;
- struct insn_emulation *insn;
- insn = kzalloc(sizeof(*insn), GFP_KERNEL);
- if (!insn)
- return;
-
- insn->ops = ops;
insn->min = INSN_UNDEF;
- switch (ops->status) {
+ switch (insn->status) {
case INSN_DEPRECATED:
insn->current_mode = INSN_EMULATE;
/* Disable the HW mode if it was turned on at early boot time */
@@ -247,7 +237,7 @@ static void __init register_insn_emulation_sysctl(void)
sysctl->mode = 0644;
sysctl->maxlen = sizeof(int);
- sysctl->procname = insn->ops->name;
+ sysctl->procname = insn->name;
sysctl->data = &insn->current_mode;
sysctl->extra1 = &insn->min;
sysctl->extra2 = &insn->max;
@@ -445,7 +435,7 @@ static struct undef_hook swp_hooks[] = {
{ }
};
-static struct insn_emulation_ops swp_ops = {
+static struct insn_emulation insn_swp = {
.name = "swp",
.status = INSN_OBSOLETE,
.hooks = swp_hooks,
@@ -532,7 +522,7 @@ static struct undef_hook cp15_barrier_hooks[] = {
{ }
};
-static struct insn_emulation_ops cp15_barrier_ops = {
+static struct insn_emulation insn_cp15_barrier = {
.name = "cp15_barrier",
.status = INSN_DEPRECATED,
.hooks = cp15_barrier_hooks,
@@ -605,7 +595,7 @@ static struct undef_hook setend_hooks[] = {
{}
};
-static struct insn_emulation_ops setend_ops = {
+static struct insn_emulation insn_setend = {
.name = "setend",
.status = INSN_DEPRECATED,
.hooks = setend_hooks,
@@ -619,14 +609,14 @@ static struct insn_emulation_ops setend_ops = {
static int __init armv8_deprecated_init(void)
{
if (IS_ENABLED(CONFIG_SWP_EMULATION))
- register_insn_emulation(&swp_ops);
+ register_insn_emulation(&insn_swp);
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
- register_insn_emulation(&cp15_barrier_ops);
+ register_insn_emulation(&insn_cp15_barrier);
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
if (system_supports_mixed_endian_el0())
- register_insn_emulation(&setend_ops);
+ register_insn_emulation(&insn_setend);
else
pr_info("setend instruction emulation is not supported on this system\n");
}
From 25eeac0cfe7c97ade1be07340e11e7143aab57a6 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:21 +0100
Subject: [PATCH 051/151] arm64: armv8_deprecated move emulation functions
Subsequent patches will rework the logic in armv8_deprecated.c.
In preparation for subsequent changes, this patch moves the emulation
logic earlier in the file, and moves the infrastructure later in the
file. This will make subsequent diffs simpler and easier to read.
This is purely a move. There should be no functional change as a result
of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-8-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/armv8_deprecated.c | 394 +++++++++++++--------------
1 file changed, 197 insertions(+), 197 deletions(-)
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 2a75e20a3913..792b89929c04 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -52,203 +52,6 @@ struct insn_emulation {
int max;
};
-static LIST_HEAD(insn_emulation);
-static int nr_insn_emulated __initdata;
-static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
-static DEFINE_MUTEX(insn_emulation_mutex);
-
-static void register_emulation_hooks(struct insn_emulation *insn)
-{
- struct undef_hook *hook;
-
- BUG_ON(!insn->hooks);
-
- for (hook = insn->hooks; hook->instr_mask; hook++)
- register_undef_hook(hook);
-
- pr_notice("Registered %s emulation handler\n", insn->name);
-}
-
-static void remove_emulation_hooks(struct insn_emulation *insn)
-{
- struct undef_hook *hook;
-
- BUG_ON(!insn->hooks);
-
- for (hook = insn->hooks; hook->instr_mask; hook++)
- unregister_undef_hook(hook);
-
- pr_notice("Removed %s emulation handler\n", insn->name);
-}
-
-static void enable_insn_hw_mode(void *data)
-{
- struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->set_hw_mode)
- insn->set_hw_mode(true);
-}
-
-static void disable_insn_hw_mode(void *data)
-{
- struct insn_emulation *insn = (struct insn_emulation *)data;
- if (insn->set_hw_mode)
- insn->set_hw_mode(false);
-}
-
-/* Run set_hw_mode(mode) on all active CPUs */
-static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
-{
- if (!insn->set_hw_mode)
- return -EINVAL;
- if (enable)
- on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
- else
- on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
- return 0;
-}
-
-/*
- * Run set_hw_mode for all insns on a starting CPU.
- * Returns:
- * 0 - If all the hooks ran successfully.
- * -EINVAL - At least one hook is not supported by the CPU.
- */
-static int run_all_insn_set_hw_mode(unsigned int cpu)
-{
- int rc = 0;
- unsigned long flags;
- struct insn_emulation *insn;
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- bool enable = (insn->current_mode == INSN_HW);
- if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
- pr_warn("CPU[%u] cannot support the emulation of %s",
- cpu, insn->name);
- rc = -EINVAL;
- }
- }
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
- return rc;
-}
-
-static int update_insn_emulation_mode(struct insn_emulation *insn,
- enum insn_emulation_mode prev)
-{
- int ret = 0;
-
- switch (prev) {
- case INSN_UNDEF: /* Nothing to be done */
- break;
- case INSN_EMULATE:
- remove_emulation_hooks(insn);
- break;
- case INSN_HW:
- if (!run_all_cpu_set_hw_mode(insn, false))
- pr_notice("Disabled %s support\n", insn->name);
- break;
- }
-
- switch (insn->current_mode) {
- case INSN_UNDEF:
- break;
- case INSN_EMULATE:
- register_emulation_hooks(insn);
- break;
- case INSN_HW:
- ret = run_all_cpu_set_hw_mode(insn, true);
- if (!ret)
- pr_notice("Enabled %s support\n", insn->name);
- break;
- }
-
- return ret;
-}
-
-static void __init register_insn_emulation(struct insn_emulation *insn)
-{
- unsigned long flags;
-
- insn->min = INSN_UNDEF;
-
- switch (insn->status) {
- case INSN_DEPRECATED:
- insn->current_mode = INSN_EMULATE;
- /* Disable the HW mode if it was turned on at early boot time */
- run_all_cpu_set_hw_mode(insn, false);
- insn->max = INSN_HW;
- break;
- case INSN_OBSOLETE:
- insn->current_mode = INSN_UNDEF;
- insn->max = INSN_EMULATE;
- break;
- }
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_add(&insn->node, &insn_emulation);
- nr_insn_emulated++;
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-
- /* Register any handlers if required */
- update_insn_emulation_mode(insn, INSN_UNDEF);
-}
-
-static int emulation_proc_handler(struct ctl_table *table, int write,
- void *buffer, size_t *lenp,
- loff_t *ppos)
-{
- int ret = 0;
- struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
- enum insn_emulation_mode prev_mode = insn->current_mode;
-
- mutex_lock(&insn_emulation_mutex);
- ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
-
- if (ret || !write || prev_mode == insn->current_mode)
- goto ret;
-
- ret = update_insn_emulation_mode(insn, prev_mode);
- if (ret) {
- /* Mode change failed, revert to previous mode. */
- insn->current_mode = prev_mode;
- update_insn_emulation_mode(insn, INSN_UNDEF);
- }
-ret:
- mutex_unlock(&insn_emulation_mutex);
- return ret;
-}
-
-static void __init register_insn_emulation_sysctl(void)
-{
- unsigned long flags;
- int i = 0;
- struct insn_emulation *insn;
- struct ctl_table *insns_sysctl, *sysctl;
-
- insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
- GFP_KERNEL);
- if (!insns_sysctl)
- return;
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- sysctl = &insns_sysctl[i];
-
- sysctl->mode = 0644;
- sysctl->maxlen = sizeof(int);
-
- sysctl->procname = insn->name;
- sysctl->data = &insn->current_mode;
- sysctl->extra1 = &insn->min;
- sysctl->extra2 = &insn->max;
- sysctl->proc_handler = emulation_proc_handler;
- i++;
- }
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-
- register_sysctl("abi", insns_sysctl);
-}
-
/*
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
* store-exclusive.
@@ -602,6 +405,203 @@ static struct insn_emulation insn_setend = {
.set_hw_mode = setend_set_hw_mode,
};
+static LIST_HEAD(insn_emulation);
+static int nr_insn_emulated __initdata;
+static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
+static DEFINE_MUTEX(insn_emulation_mutex);
+
+static void register_emulation_hooks(struct insn_emulation *insn)
+{
+ struct undef_hook *hook;
+
+ BUG_ON(!insn->hooks);
+
+ for (hook = insn->hooks; hook->instr_mask; hook++)
+ register_undef_hook(hook);
+
+ pr_notice("Registered %s emulation handler\n", insn->name);
+}
+
+static void remove_emulation_hooks(struct insn_emulation *insn)
+{
+ struct undef_hook *hook;
+
+ BUG_ON(!insn->hooks);
+
+ for (hook = insn->hooks; hook->instr_mask; hook++)
+ unregister_undef_hook(hook);
+
+ pr_notice("Removed %s emulation handler\n", insn->name);
+}
+
+static void enable_insn_hw_mode(void *data)
+{
+ struct insn_emulation *insn = (struct insn_emulation *)data;
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(true);
+}
+
+static void disable_insn_hw_mode(void *data)
+{
+ struct insn_emulation *insn = (struct insn_emulation *)data;
+ if (insn->set_hw_mode)
+ insn->set_hw_mode(false);
+}
+
+/* Run set_hw_mode(mode) on all active CPUs */
+static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
+{
+ if (!insn->set_hw_mode)
+ return -EINVAL;
+ if (enable)
+ on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
+ else
+ on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
+ return 0;
+}
+
+/*
+ * Run set_hw_mode for all insns on a starting CPU.
+ * Returns:
+ * 0 - If all the hooks ran successfully.
+ * -EINVAL - At least one hook is not supported by the CPU.
+ */
+static int run_all_insn_set_hw_mode(unsigned int cpu)
+{
+ int rc = 0;
+ unsigned long flags;
+ struct insn_emulation *insn;
+
+ raw_spin_lock_irqsave(&insn_emulation_lock, flags);
+ list_for_each_entry(insn, &insn_emulation, node) {
+ bool enable = (insn->current_mode == INSN_HW);
+ if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
+ pr_warn("CPU[%u] cannot support the emulation of %s",
+ cpu, insn->name);
+ rc = -EINVAL;
+ }
+ }
+ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
+ return rc;
+}
+
+static int update_insn_emulation_mode(struct insn_emulation *insn,
+ enum insn_emulation_mode prev)
+{
+ int ret = 0;
+
+ switch (prev) {
+ case INSN_UNDEF: /* Nothing to be done */
+ break;
+ case INSN_EMULATE:
+ remove_emulation_hooks(insn);
+ break;
+ case INSN_HW:
+ if (!run_all_cpu_set_hw_mode(insn, false))
+ pr_notice("Disabled %s support\n", insn->name);
+ break;
+ }
+
+ switch (insn->current_mode) {
+ case INSN_UNDEF:
+ break;
+ case INSN_EMULATE:
+ register_emulation_hooks(insn);
+ break;
+ case INSN_HW:
+ ret = run_all_cpu_set_hw_mode(insn, true);
+ if (!ret)
+ pr_notice("Enabled %s support\n", insn->name);
+ break;
+ }
+
+ return ret;
+}
+
+static void __init register_insn_emulation(struct insn_emulation *insn)
+{
+ unsigned long flags;
+
+ insn->min = INSN_UNDEF;
+
+ switch (insn->status) {
+ case INSN_DEPRECATED:
+ insn->current_mode = INSN_EMULATE;
+ /* Disable the HW mode if it was turned on at early boot time */
+ run_all_cpu_set_hw_mode(insn, false);
+ insn->max = INSN_HW;
+ break;
+ case INSN_OBSOLETE:
+ insn->current_mode = INSN_UNDEF;
+ insn->max = INSN_EMULATE;
+ break;
+ }
+
+ raw_spin_lock_irqsave(&insn_emulation_lock, flags);
+ list_add(&insn->node, &insn_emulation);
+ nr_insn_emulated++;
+ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
+
+ /* Register any handlers if required */
+ update_insn_emulation_mode(insn, INSN_UNDEF);
+}
+
+static int emulation_proc_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *lenp,
+ loff_t *ppos)
+{
+ int ret = 0;
+ struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
+ enum insn_emulation_mode prev_mode = insn->current_mode;
+
+ mutex_lock(&insn_emulation_mutex);
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
+ if (ret || !write || prev_mode == insn->current_mode)
+ goto ret;
+
+ ret = update_insn_emulation_mode(insn, prev_mode);
+ if (ret) {
+ /* Mode change failed, revert to previous mode. */
+ insn->current_mode = prev_mode;
+ update_insn_emulation_mode(insn, INSN_UNDEF);
+ }
+ret:
+ mutex_unlock(&insn_emulation_mutex);
+ return ret;
+}
+
+static void __init register_insn_emulation_sysctl(void)
+{
+ unsigned long flags;
+ int i = 0;
+ struct insn_emulation *insn;
+ struct ctl_table *insns_sysctl, *sysctl;
+
+ insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
+ GFP_KERNEL);
+ if (!insns_sysctl)
+ return;
+
+ raw_spin_lock_irqsave(&insn_emulation_lock, flags);
+ list_for_each_entry(insn, &insn_emulation, node) {
+ sysctl = &insns_sysctl[i];
+
+ sysctl->mode = 0644;
+ sysctl->maxlen = sizeof(int);
+
+ sysctl->procname = insn->name;
+ sysctl->data = &insn->current_mode;
+ sysctl->extra1 = &insn->min;
+ sysctl->extra2 = &insn->max;
+ sysctl->proc_handler = emulation_proc_handler;
+ i++;
+ }
+ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
+
+ register_sysctl("abi", insns_sysctl);
+}
+
/*
* Invoked as core_initcall, which guarantees that the instruction
* emulation is ready for userspace.
From 0c5f416219da3795dc8b33e5bb7865a6b3c4e55c Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:22 +0100
Subject: [PATCH 052/151] arm64: armv8_deprecated: move aarch32 helper earlier
Subsequent patches will rework the logic in armv8_deprecated.c.
In preparation for subsequent changes, this patch moves some shared logic
earlier in the file. This will make subsequent diffs simpler and easier to
read.
At the same time, drop the `__kprobes` annotation from
aarch32_check_condition(), as this is only used for traps from compat
userspace, and has no risk of recursion within kprobes. As this is the
last kprobes annotation in armve8_deprecated.c, we no longer need to
include <asm/kprobes.h>.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-9-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/armv8_deprecated.c | 39 ++++++++++++++--------------
1 file changed, 19 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 792b89929c04..7f2ce49dbf97 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -17,7 +17,6 @@
#include <asm/sysreg.h>
#include <asm/system_misc.h>
#include <asm/traps.h>
-#include <asm/kprobes.h>
#define CREATE_TRACE_POINTS
#include "trace-events-emulation.h"
@@ -52,6 +51,25 @@ struct insn_emulation {
int max;
};
+#define ARM_OPCODE_CONDTEST_FAIL 0
+#define ARM_OPCODE_CONDTEST_PASS 1
+#define ARM_OPCODE_CONDTEST_UNCOND 2
+
+#define ARM_OPCODE_CONDITION_UNCOND 0xf
+
+static unsigned int aarch32_check_condition(u32 opcode, u32 psr)
+{
+ u32 cc_bits = opcode >> 28;
+
+ if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
+ if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
+ return ARM_OPCODE_CONDTEST_PASS;
+ else
+ return ARM_OPCODE_CONDTEST_FAIL;
+ }
+ return ARM_OPCODE_CONDTEST_UNCOND;
+}
+
/*
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
* store-exclusive.
@@ -132,25 +150,6 @@ static int emulate_swpX(unsigned int address, unsigned int *data,
return res;
}
-#define ARM_OPCODE_CONDTEST_FAIL 0
-#define ARM_OPCODE_CONDTEST_PASS 1
-#define ARM_OPCODE_CONDTEST_UNCOND 2
-
-#define ARM_OPCODE_CONDITION_UNCOND 0xf
-
-static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
-{
- u32 cc_bits = opcode >> 28;
-
- if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
- if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
- return ARM_OPCODE_CONDTEST_PASS;
- else
- return ARM_OPCODE_CONDTEST_FAIL;
- }
- return ARM_OPCODE_CONDTEST_UNCOND;
-}
-
/*
* swp_handler logs the id of calling process, dissects the instruction, sanity
* checks the memory location, calls emulate_swpX for the actual operation and
From 124c49b1b5d947b7180c5d6cbb09ddf76ea45ea2 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Wed, 19 Oct 2022 15:41:23 +0100
Subject: [PATCH 053/151] arm64: armv8_deprecated: rework deprected instruction
handling
Support for deprecated instructions can be enabled or disabled at
runtime. To handle this, the code in armv8_deprecated.c registers and
unregisters undef_hooks, and makes cross CPU calls to configure HW
support. This is rather complicated, and the synchronization required to
make this safe ends up serializing the handling of instructions which
have been trapped.
This patch simplifies the deprecated instruction handling by removing
the dynamic registration and unregistration, and changing the trap
handling code to determine whether a handler should be invoked. This
removes the need for dynamic list management, and simplifies the locking
requirements, making it possible to handle trapped instructions entirely
in parallel.
Where changing the emulation state requires a cross-call, this is
serialized by locally disabling interrupts, ensuring that the CPU is not
left in an inconsistent state.
To simplify sysctl management, each insn_emulation is given a separate
sysctl table, permitting these to be registered separately. The core
sysctl code will iterate over all of these when walking sysfs.
I've tested this with userspace programs which use each of the
deprecated instructions, and I've concurrently modified the support
level for each of the features back-and-forth between HW and emulated to
check that there are no spurious SIGILLs sent to userspace when the
support level is changed.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221019144123.612388-10-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/traps.h | 19 +-
arch/arm64/kernel/armv8_deprecated.c | 284 +++++++++++++--------------
arch/arm64/kernel/traps.c | 40 +---
3 files changed, 150 insertions(+), 193 deletions(-)
diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h
index 6e5826470bea..1f361e2da516 100644
--- a/arch/arm64/include/asm/traps.h
+++ b/arch/arm64/include/asm/traps.h
@@ -13,17 +13,16 @@
struct pt_regs;
-struct undef_hook {
- struct list_head node;
- u32 instr_mask;
- u32 instr_val;
- u64 pstate_mask;
- u64 pstate_val;
- int (*fn)(struct pt_regs *regs, u32 instr);
-};
+#ifdef CONFIG_ARMV8_DEPRECATED
+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
+#else
+static inline bool
+try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
+{
+ return false;
+}
+#endif /* CONFIG_ARMV8_DEPRECATED */
-void register_undef_hook(struct undef_hook *hook);
-void unregister_undef_hook(struct undef_hook *hook);
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err);
void arm64_notify_segfault(unsigned long addr);
void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str);
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 7f2ce49dbf97..ed0788cf6bbb 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -38,17 +38,24 @@ enum insn_emulation_mode {
enum legacy_insn_status {
INSN_DEPRECATED,
INSN_OBSOLETE,
+ INSN_UNAVAILABLE,
};
struct insn_emulation {
const char *name;
- struct list_head node;
enum legacy_insn_status status;
- struct undef_hook *hooks;
+ bool (*try_emulate)(struct pt_regs *regs,
+ u32 insn);
int (*set_hw_mode)(bool enable);
+
int current_mode;
int min;
int max;
+
+ /*
+ * sysctl for this emulation + a sentinal entry.
+ */
+ struct ctl_table sysctl[2];
};
#define ARM_OPCODE_CONDTEST_FAIL 0
@@ -70,6 +77,7 @@ static unsigned int aarch32_check_condition(u32 opcode, u32 psr)
return ARM_OPCODE_CONDTEST_UNCOND;
}
+#ifdef CONFIG_SWP_EMULATION
/*
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
* store-exclusive.
@@ -222,28 +230,27 @@ fault:
return 0;
}
-/*
- * Only emulate SWP/SWPB executed in ARM state/User mode.
- * The kernel must be SWP free and SWP{B} does not exist in Thumb.
- */
-static struct undef_hook swp_hooks[] = {
- {
- .instr_mask = 0x0fb00ff0,
- .instr_val = 0x01000090,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = swp_handler
- },
- { }
-};
+static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
+{
+ /* SWP{B} only exists in ARM state and does not exist in Thumb */
+ if (!compat_user_mode(regs) || compat_thumb_mode(regs))
+ return false;
+
+ if ((insn & 0x0fb00ff0) != 0x01000090)
+ return false;
+
+ return swp_handler(regs, insn) == 0;
+}
static struct insn_emulation insn_swp = {
.name = "swp",
.status = INSN_OBSOLETE,
- .hooks = swp_hooks,
+ .try_emulate = try_emulate_swp,
.set_hw_mode = NULL,
};
+#endif /* CONFIG_SWP_EMULATION */
+#ifdef CONFIG_CP15_BARRIER_EMULATION
static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
{
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
@@ -306,31 +313,29 @@ static int cp15_barrier_set_hw_mode(bool enable)
return 0;
}
-static struct undef_hook cp15_barrier_hooks[] = {
- {
- .instr_mask = 0x0fff0fdf,
- .instr_val = 0x0e070f9a,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = cp15barrier_handler,
- },
- {
- .instr_mask = 0x0fff0fff,
- .instr_val = 0x0e070f95,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = cp15barrier_handler,
- },
- { }
-};
+static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
+{
+ if (!compat_user_mode(regs) || compat_thumb_mode(regs))
+ return false;
+
+ if ((insn & 0x0fff0fdf) == 0x0e070f9a)
+ return cp15barrier_handler(regs, insn) == 0;
+
+ if ((insn & 0x0fff0fff) == 0x0e070f95)
+ return cp15barrier_handler(regs, insn) == 0;
+
+ return false;
+}
static struct insn_emulation insn_cp15_barrier = {
.name = "cp15_barrier",
.status = INSN_DEPRECATED,
- .hooks = cp15_barrier_hooks,
+ .try_emulate = try_emulate_cp15_barrier,
.set_hw_mode = cp15_barrier_set_hw_mode,
};
+#endif /* CONFIG_CP15_BARRIER_EMULATION */
+#ifdef CONFIG_SETEND_EMULATION
static int setend_set_hw_mode(bool enable)
{
if (!cpu_supports_mixed_endian_el0())
@@ -378,61 +383,41 @@ static int t16_setend_handler(struct pt_regs *regs, u32 instr)
return rc;
}
-static struct undef_hook setend_hooks[] = {
- {
- .instr_mask = 0xfffffdff,
- .instr_val = 0xf1010000,
- .pstate_mask = PSR_AA32_MODE_MASK,
- .pstate_val = PSR_AA32_MODE_USR,
- .fn = a32_setend_handler,
- },
- {
- /* Thumb mode */
- .instr_mask = 0xfffffff7,
- .instr_val = 0x0000b650,
- .pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK),
- .pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR),
- .fn = t16_setend_handler,
- },
- {}
-};
+static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
+{
+ if (compat_thumb_mode(regs) &&
+ (insn & 0xfffffff7) == 0x0000b650)
+ return t16_setend_handler(regs, insn) == 0;
+
+ if (compat_user_mode(regs) &&
+ (insn & 0xfffffdff) == 0xf1010000)
+ return a32_setend_handler(regs, insn) == 0;
+
+ return false;
+}
static struct insn_emulation insn_setend = {
.name = "setend",
.status = INSN_DEPRECATED,
- .hooks = setend_hooks,
+ .try_emulate = try_emulate_setend,
.set_hw_mode = setend_set_hw_mode,
};
+#endif /* CONFIG_SETEND_EMULATION */
+
+static struct insn_emulation *insn_emulations[] = {
+#ifdef CONFIG_SWP_EMULATION
+ &insn_swp,
+#endif
+#ifdef CONFIG_CP15_BARRIER_EMULATION
+ &insn_cp15_barrier,
+#endif
+#ifdef CONFIG_SETEND_EMULATION
+ &insn_setend,
+#endif
+};
-static LIST_HEAD(insn_emulation);
-static int nr_insn_emulated __initdata;
-static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
static DEFINE_MUTEX(insn_emulation_mutex);
-static void register_emulation_hooks(struct insn_emulation *insn)
-{
- struct undef_hook *hook;
-
- BUG_ON(!insn->hooks);
-
- for (hook = insn->hooks; hook->instr_mask; hook++)
- register_undef_hook(hook);
-
- pr_notice("Registered %s emulation handler\n", insn->name);
-}
-
-static void remove_emulation_hooks(struct insn_emulation *insn)
-{
- struct undef_hook *hook;
-
- BUG_ON(!insn->hooks);
-
- for (hook = insn->hooks; hook->instr_mask; hook++)
- unregister_undef_hook(hook);
-
- pr_notice("Removed %s emulation handler\n", insn->name);
-}
-
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
@@ -469,18 +454,24 @@ static int run_all_insn_set_hw_mode(unsigned int cpu)
{
int rc = 0;
unsigned long flags;
- struct insn_emulation *insn;
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- bool enable = (insn->current_mode == INSN_HW);
+ /*
+ * Disable IRQs to serialize against an IPI from
+ * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
+ * recent enablement state if the two race with one another.
+ */
+ local_irq_save(flags);
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *insn = insn_emulations[i];
+ bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
pr_warn("CPU[%u] cannot support the emulation of %s",
cpu, insn->name);
rc = -EINVAL;
}
}
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
+ local_irq_restore(flags);
+
return rc;
}
@@ -493,7 +484,6 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
case INSN_UNDEF: /* Nothing to be done */
break;
case INSN_EMULATE:
- remove_emulation_hooks(insn);
break;
case INSN_HW:
if (!run_all_cpu_set_hw_mode(insn, false))
@@ -505,7 +495,6 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
case INSN_UNDEF:
break;
case INSN_EMULATE:
- register_emulation_hooks(insn);
break;
case INSN_HW:
ret = run_all_cpu_set_hw_mode(insn, true);
@@ -517,34 +506,6 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
return ret;
}
-static void __init register_insn_emulation(struct insn_emulation *insn)
-{
- unsigned long flags;
-
- insn->min = INSN_UNDEF;
-
- switch (insn->status) {
- case INSN_DEPRECATED:
- insn->current_mode = INSN_EMULATE;
- /* Disable the HW mode if it was turned on at early boot time */
- run_all_cpu_set_hw_mode(insn, false);
- insn->max = INSN_HW;
- break;
- case INSN_OBSOLETE:
- insn->current_mode = INSN_UNDEF;
- insn->max = INSN_EMULATE;
- break;
- }
-
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_add(&insn->node, &insn_emulation);
- nr_insn_emulated++;
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-
- /* Register any handlers if required */
- update_insn_emulation_mode(insn, INSN_UNDEF);
-}
-
static int emulation_proc_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp,
loff_t *ppos)
@@ -562,7 +523,7 @@ static int emulation_proc_handler(struct ctl_table *table, int write,
ret = update_insn_emulation_mode(insn, prev_mode);
if (ret) {
/* Mode change failed, revert to previous mode. */
- insn->current_mode = prev_mode;
+ WRITE_ONCE(insn->current_mode, prev_mode);
update_insn_emulation_mode(insn, INSN_UNDEF);
}
ret:
@@ -570,21 +531,34 @@ ret:
return ret;
}
-static void __init register_insn_emulation_sysctl(void)
+static void __init register_insn_emulation(struct insn_emulation *insn)
{
- unsigned long flags;
- int i = 0;
- struct insn_emulation *insn;
- struct ctl_table *insns_sysctl, *sysctl;
+ struct ctl_table *sysctl;
- insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
- GFP_KERNEL);
- if (!insns_sysctl)
- return;
+ insn->min = INSN_UNDEF;
- raw_spin_lock_irqsave(&insn_emulation_lock, flags);
- list_for_each_entry(insn, &insn_emulation, node) {
- sysctl = &insns_sysctl[i];
+ switch (insn->status) {
+ case INSN_DEPRECATED:
+ insn->current_mode = INSN_EMULATE;
+ /* Disable the HW mode if it was turned on at early boot time */
+ run_all_cpu_set_hw_mode(insn, false);
+ insn->max = INSN_HW;
+ break;
+ case INSN_OBSOLETE:
+ insn->current_mode = INSN_UNDEF;
+ insn->max = INSN_EMULATE;
+ break;
+ case INSN_UNAVAILABLE:
+ insn->current_mode = INSN_UNDEF;
+ insn->max = INSN_UNDEF;
+ break;
+ }
+
+ /* Program the HW if required */
+ update_insn_emulation_mode(insn, INSN_UNDEF);
+
+ if (insn->status != INSN_UNAVAILABLE) {
+ sysctl = &insn->sysctl[0];
sysctl->mode = 0644;
sysctl->maxlen = sizeof(int);
@@ -594,11 +568,32 @@ static void __init register_insn_emulation_sysctl(void)
sysctl->extra1 = &insn->min;
sysctl->extra2 = &insn->max;
sysctl->proc_handler = emulation_proc_handler;
- i++;
- }
- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
- register_sysctl("abi", insns_sysctl);
+ register_sysctl("abi", sysctl);
+ }
+}
+
+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
+{
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *ie = insn_emulations[i];
+
+ if (ie->status == INSN_UNAVAILABLE)
+ continue;
+
+ /*
+ * A trap may race with the mode being changed
+ * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
+ * avoid a spurious UNDEF.
+ */
+ if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
+ continue;
+
+ if (ie->try_emulate(regs, insn))
+ return true;
+ }
+
+ return false;
}
/*
@@ -607,24 +602,25 @@ static void __init register_insn_emulation_sysctl(void)
*/
static int __init armv8_deprecated_init(void)
{
- if (IS_ENABLED(CONFIG_SWP_EMULATION))
- register_insn_emulation(&insn_swp);
+#ifdef CONFIG_SETEND_EMULATION
+ if (!system_supports_mixed_endian_el0()) {
+ insn_setend.status = INSN_UNAVAILABLE;
+ pr_info("setend instruction emulation is not supported on this system\n");
+ }
- if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
- register_insn_emulation(&insn_cp15_barrier);
+#endif
+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
+ struct insn_emulation *ie = insn_emulations[i];
- if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
- if (system_supports_mixed_endian_el0())
- register_insn_emulation(&insn_setend);
- else
- pr_info("setend instruction emulation is not supported on this system\n");
+ if (ie->status == INSN_UNAVAILABLE)
+ continue;
+
+ register_insn_emulation(ie);
}
cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
"arm64/isndep:starting",
run_all_insn_set_hw_mode, NULL);
- register_insn_emulation_sysctl();
-
return 0;
}
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 96eaf1aaec12..4c0caa589e12 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -373,27 +373,6 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
regs->pstate &= ~PSR_BTYPE_MASK;
}
-static LIST_HEAD(undef_hook);
-static DEFINE_RAW_SPINLOCK(undef_lock);
-
-void register_undef_hook(struct undef_hook *hook)
-{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_add(&hook->node, &undef_hook);
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-}
-
-void unregister_undef_hook(struct undef_hook *hook)
-{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_del(&hook->node);
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-}
-
static int user_insn_read(struct pt_regs *regs, u32 *insnp)
{
u32 instr;
@@ -425,23 +404,6 @@ static int user_insn_read(struct pt_regs *regs, u32 *insnp)
return 0;
}
-static int call_undef_hook(struct pt_regs *regs, u32 instr)
-{
- struct undef_hook *hook;
- unsigned long flags;
- int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
-
- raw_spin_lock_irqsave(&undef_lock, flags);
- list_for_each_entry(hook, &undef_hook, node)
- if ((instr & hook->instr_mask) == hook->instr_val &&
- (regs->pstate & hook->pstate_mask) == hook->pstate_val)
- fn = hook->fn;
-
- raw_spin_unlock_irqrestore(&undef_lock, flags);
-
- return fn ? fn(regs, instr) : 1;
-}
-
void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
{
const char *desc;
@@ -502,7 +464,7 @@ void do_el0_undef(struct pt_regs *regs, unsigned long esr)
if (try_emulate_mrs(regs, insn))
return;
- if (call_undef_hook(regs, insn) == 0)
+ if (try_emulate_armv8_deprecated(regs, insn))
return;
out_err:
From d9f564c966e63925aac4ba273a9319d7fb6f4b4e Mon Sep 17 00:00:00 2001
From: Shang XiaoJing <shangxiaojing@huawei.com>
Date: Tue, 15 Nov 2022 19:55:39 +0800
Subject: [PATCH 054/151] perf/arm_dmc620: Fix hotplug callback leak in
dmc620_pmu_init()
dmc620_pmu_init() won't remove the callback added by
cpuhp_setup_state_multi() when platform_driver_register() failed. Remove
the callback by cpuhp_remove_multi_state() in fail path.
Similar to the handling of arm_ccn_init() in commit 26242b330093 ("bus:
arm-ccn: Prevent hotplug callback leak")
Fixes: 53c218da220c ("driver/perf: Add PMU driver for the ARM DMC-620 memory controller")
Signed-off-by: Shang XiaoJing <shangxiaojing@huawei.com>
Reviewed-by: Punit Agrawal <punit.agrawal@bytedance.com>
Link: https://lore.kernel.org/r/20221115115540.6245-2-shangxiaojing@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_dmc620_pmu.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c
index 280a6ae3e27c..54aa4658fb36 100644
--- a/drivers/perf/arm_dmc620_pmu.c
+++ b/drivers/perf/arm_dmc620_pmu.c
@@ -725,6 +725,8 @@ static struct platform_driver dmc620_pmu_driver = {
static int __init dmc620_pmu_init(void)
{
+ int ret;
+
cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
DMC620_DRVNAME,
NULL,
@@ -732,7 +734,11 @@ static int __init dmc620_pmu_init(void)
if (cpuhp_state_num < 0)
return cpuhp_state_num;
- return platform_driver_register(&dmc620_pmu_driver);
+ ret = platform_driver_register(&dmc620_pmu_driver);
+ if (ret)
+ cpuhp_remove_multi_state(cpuhp_state_num);
+
+ return ret;
}
static void __exit dmc620_pmu_exit(void)
From 6f2d566b46436a50a80d6445e82879686b89588c Mon Sep 17 00:00:00 2001
From: Shang XiaoJing <shangxiaojing@huawei.com>
Date: Tue, 15 Nov 2022 19:55:40 +0800
Subject: [PATCH 055/151] perf/smmuv3: Fix hotplug callback leak in
arm_smmu_pmu_init()
arm_smmu_pmu_init() won't remove the callback added by
cpuhp_setup_state_multi() when platform_driver_register() failed. Remove
the callback by cpuhp_remove_multi_state() in fail path.
Similar to the handling of arm_ccn_init() in commit 26242b330093 ("bus:
arm-ccn: Prevent hotplug callback leak")
Fixes: 7d839b4b9e00 ("perf/smmuv3: Add arm64 smmuv3 pmu driver")
Signed-off-by: Shang XiaoJing <shangxiaojing@huawei.com>
Reviewed-by: Punit Agrawal <punit.agrawal@bytedance.com>
Link: https://lore.kernel.org/r/20221115115540.6245-3-shangxiaojing@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_smmuv3_pmu.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 00d4c45a8017..25a269d431e4 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -959,6 +959,8 @@ static struct platform_driver smmu_pmu_driver = {
static int __init arm_smmu_pmu_init(void)
{
+ int ret;
+
cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
"perf/arm/pmcg:online",
NULL,
@@ -966,7 +968,11 @@ static int __init arm_smmu_pmu_init(void)
if (cpuhp_state_num < 0)
return cpuhp_state_num;
- return platform_driver_register(&smmu_pmu_driver);
+ ret = platform_driver_register(&smmu_pmu_driver);
+ if (ret)
+ cpuhp_remove_multi_state(cpuhp_state_num);
+
+ return ret;
}
module_init(arm_smmu_pmu_init);
From e37dfd65731dc4f001fa7dfa7f705e6840017d5a Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Fri, 11 Nov 2022 16:23:28 -0600
Subject: [PATCH 056/151] perf: arm_cspmu: Add support for ARM CoreSight PMU
driver
Add support for ARM CoreSight PMU driver framework and interfaces.
The driver provides generic implementation to operate uncore PMU based
on ARM CoreSight PMU architecture. The driver also provides interface
to get vendor/implementation specific information, for example event
attributes and formating.
The specification used in this implementation can be found below:
* ACPI Arm Performance Monitoring Unit table:
https://developer.arm.com/documentation/den0117/latest
* ARM Coresight PMU architecture:
https://developer.arm.com/documentation/ihi0091/latest
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Link: https://lore.kernel.org/r/20221111222330.48602-2-bwicaksono@nvidia.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/Kconfig | 2 +
drivers/perf/Makefile | 1 +
drivers/perf/arm_cspmu/Kconfig | 13 +
drivers/perf/arm_cspmu/Makefile | 6 +
drivers/perf/arm_cspmu/arm_cspmu.c | 1292 ++++++++++++++++++++++++++++
drivers/perf/arm_cspmu/arm_cspmu.h | 151 ++++
6 files changed, 1465 insertions(+)
create mode 100644 drivers/perf/arm_cspmu/Kconfig
create mode 100644 drivers/perf/arm_cspmu/Makefile
create mode 100644 drivers/perf/arm_cspmu/arm_cspmu.c
create mode 100644 drivers/perf/arm_cspmu/arm_cspmu.h
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 341010f20b77..fa87dedf8c92 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -199,4 +199,6 @@ config MARVELL_CN10K_DDR_PMU
Enable perf support for Marvell DDR Performance monitoring
event on CN10K platform.
+source "drivers/perf/arm_cspmu/Kconfig"
+
endmenu
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 050d04ee19dd..35bb0d979a70 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -21,3 +21,4 @@ obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o
obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o
obj-$(CONFIG_APPLE_M1_CPU_PMU) += apple_m1_cpu_pmu.o
obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o
+obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu/
diff --git a/drivers/perf/arm_cspmu/Kconfig b/drivers/perf/arm_cspmu/Kconfig
new file mode 100644
index 000000000000..058223bef661
--- /dev/null
+++ b/drivers/perf/arm_cspmu/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+config ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU
+ tristate "ARM Coresight Architecture PMU"
+ depends on ACPI
+ depends on ACPI_APMT || COMPILE_TEST
+ help
+ Provides support for performance monitoring unit (PMU) devices
+ based on ARM CoreSight PMU architecture. Note that this PMU
+ architecture does not have relationship with the ARM CoreSight
+ Self-Hosted Tracing.
diff --git a/drivers/perf/arm_cspmu/Makefile b/drivers/perf/arm_cspmu/Makefile
new file mode 100644
index 000000000000..9bc76de4002d
--- /dev/null
+++ b/drivers/perf/arm_cspmu/Makefile
@@ -0,0 +1,6 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += \
+ arm_cspmu.o
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c
new file mode 100644
index 000000000000..e8f57aa9f047
--- /dev/null
+++ b/drivers/perf/arm_cspmu/arm_cspmu.c
@@ -0,0 +1,1292 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ARM CoreSight Architecture PMU driver.
+ *
+ * This driver adds support for uncore PMU based on ARM CoreSight Performance
+ * Monitoring Unit Architecture. The PMU is accessible via MMIO registers and
+ * like other uncore PMUs, it does not support process specific events and
+ * cannot be used in sampling mode.
+ *
+ * This code is based on other uncore PMUs like ARM DSU PMU. It provides a
+ * generic implementation to operate the PMU according to CoreSight PMU
+ * architecture and ACPI ARM PMU table (APMT) documents below:
+ * - ARM CoreSight PMU architecture document number: ARM IHI 0091 A.a-00bet0.
+ * - APMT document number: ARM DEN0117.
+ *
+ * The user should refer to the vendor technical documentation to get details
+ * about the supported events.
+ *
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ */
+
+#include <linux/acpi.h>
+#include <linux/cacheinfo.h>
+#include <linux/ctype.h>
+#include <linux/interrupt.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/module.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <acpi/processor.h>
+
+#include "arm_cspmu.h"
+
+#define PMUNAME "arm_cspmu"
+#define DRVNAME "arm-cs-arch-pmu"
+
+#define ARM_CSPMU_CPUMASK_ATTR(_name, _config) \
+ ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_cpumask_show, \
+ (unsigned long)_config)
+
+/*
+ * CoreSight PMU Arch register offsets.
+ */
+#define PMEVCNTR_LO 0x0
+#define PMEVCNTR_HI 0x4
+#define PMEVTYPER 0x400
+#define PMCCFILTR 0x47C
+#define PMEVFILTR 0xA00
+#define PMCNTENSET 0xC00
+#define PMCNTENCLR 0xC20
+#define PMINTENSET 0xC40
+#define PMINTENCLR 0xC60
+#define PMOVSCLR 0xC80
+#define PMOVSSET 0xCC0
+#define PMCFGR 0xE00
+#define PMCR 0xE04
+#define PMIIDR 0xE08
+
+/* PMCFGR register field */
+#define PMCFGR_NCG GENMASK(31, 28)
+#define PMCFGR_HDBG BIT(24)
+#define PMCFGR_TRO BIT(23)
+#define PMCFGR_SS BIT(22)
+#define PMCFGR_FZO BIT(21)
+#define PMCFGR_MSI BIT(20)
+#define PMCFGR_UEN BIT(19)
+#define PMCFGR_NA BIT(17)
+#define PMCFGR_EX BIT(16)
+#define PMCFGR_CCD BIT(15)
+#define PMCFGR_CC BIT(14)
+#define PMCFGR_SIZE GENMASK(13, 8)
+#define PMCFGR_N GENMASK(7, 0)
+
+/* PMCR register field */
+#define PMCR_TRO BIT(11)
+#define PMCR_HDBG BIT(10)
+#define PMCR_FZO BIT(9)
+#define PMCR_NA BIT(8)
+#define PMCR_DP BIT(5)
+#define PMCR_X BIT(4)
+#define PMCR_D BIT(3)
+#define PMCR_C BIT(2)
+#define PMCR_P BIT(1)
+#define PMCR_E BIT(0)
+
+/* Each SET/CLR register supports up to 32 counters. */
+#define ARM_CSPMU_SET_CLR_COUNTER_SHIFT 5
+#define ARM_CSPMU_SET_CLR_COUNTER_NUM \
+ (1 << ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
+
+/* Convert counter idx into SET/CLR register number. */
+#define COUNTER_TO_SET_CLR_ID(idx) \
+ (idx >> ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
+
+/* Convert counter idx into SET/CLR register bit. */
+#define COUNTER_TO_SET_CLR_BIT(idx) \
+ (idx & (ARM_CSPMU_SET_CLR_COUNTER_NUM - 1))
+
+#define ARM_CSPMU_ACTIVE_CPU_MASK 0x0
+#define ARM_CSPMU_ASSOCIATED_CPU_MASK 0x1
+
+/* Check if field f in flags is set with value v */
+#define CHECK_APMT_FLAG(flags, f, v) \
+ ((flags & (ACPI_APMT_FLAGS_ ## f)) == (ACPI_APMT_FLAGS_ ## f ## _ ## v))
+
+/* Check and use default if implementer doesn't provide attribute callback */
+#define CHECK_DEFAULT_IMPL_OPS(ops, callback) \
+ do { \
+ if (!ops->callback) \
+ ops->callback = arm_cspmu_ ## callback; \
+ } while (0)
+
+/*
+ * Maximum poll count for reading counter value using high-low-high sequence.
+ */
+#define HILOHI_MAX_POLL 1000
+
+static unsigned long arm_cspmu_cpuhp_state;
+
+/*
+ * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except
+ * counter register. The counter register can be implemented as 32-bit or 64-bit
+ * register depending on the value of PMCFGR.SIZE field. For 64-bit access,
+ * single-copy 64-bit atomic support is implementation defined. APMT node flag
+ * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit
+ * single copy atomic is not supported, the driver treats the register as a pair
+ * of 32-bit register.
+ */
+
+/*
+ * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
+ */
+static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
+{
+ u32 val_lo, val_hi;
+ u64 val;
+
+ /* Use high-low-high sequence to avoid tearing */
+ do {
+ if (max_poll_count-- == 0) {
+ pr_err("ARM CSPMU: timeout hi-low-high sequence\n");
+ return 0;
+ }
+
+ val_hi = readl(addr + 4);
+ val_lo = readl(addr);
+ } while (val_hi != readl(addr + 4));
+
+ val = (((u64)val_hi << 32) | val_lo);
+
+ return val;
+}
+
+/* Check if PMU supports 64-bit single copy atomic. */
+static inline bool supports_64bit_atomics(const struct arm_cspmu *cspmu)
+{
+ return CHECK_APMT_FLAG(cspmu->apmt_node->flags, ATOMIC, SUPP);
+}
+
+/* Check if cycle counter is supported. */
+static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)
+{
+ return (cspmu->pmcfgr & PMCFGR_CC);
+}
+
+/* Get counter size, which is (PMCFGR_SIZE + 1). */
+static inline u32 counter_size(const struct arm_cspmu *cspmu)
+{
+ return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;
+}
+
+/* Get counter mask. */
+static inline u64 counter_mask(const struct arm_cspmu *cspmu)
+{
+ return GENMASK_ULL(counter_size(cspmu) - 1, 0);
+}
+
+/* Check if counter is implemented as 64-bit register. */
+static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)
+{
+ return (counter_size(cspmu) > 32);
+}
+
+ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_ext_attribute *eattr =
+ container_of(attr, struct dev_ext_attribute, attr);
+ return sysfs_emit(buf, "event=0x%llx\n",
+ (unsigned long long)eattr->var);
+}
+EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_event_show);
+
+/* Default event list. */
+static struct attribute *arm_cspmu_event_attrs[] = {
+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
+ NULL,
+};
+
+static struct attribute **
+arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
+{
+ struct attribute **attrs;
+
+ attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,
+ sizeof(arm_cspmu_event_attrs), GFP_KERNEL);
+
+ return attrs;
+}
+
+static umode_t
+arm_cspmu_event_attr_is_visible(struct kobject *kobj,
+ struct attribute *attr, int unused)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
+ struct perf_pmu_events_attr *eattr;
+
+ eattr = container_of(attr, typeof(*eattr), attr.attr);
+
+ /* Hide cycle event if not supported */
+ if (!supports_cycle_counter(cspmu) &&
+ eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT)
+ return 0;
+
+ return attr->mode;
+}
+
+ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct dev_ext_attribute *eattr =
+ container_of(attr, struct dev_ext_attribute, attr);
+ return sysfs_emit(buf, "%s\n", (char *)eattr->var);
+}
+EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_format_show);
+
+static struct attribute *arm_cspmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ ARM_CSPMU_FORMAT_FILTER_ATTR,
+ NULL,
+};
+
+static struct attribute **
+arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
+{
+ struct attribute **attrs;
+
+ attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,
+ sizeof(arm_cspmu_format_attrs), GFP_KERNEL);
+
+ return attrs;
+}
+
+static u32 arm_cspmu_event_type(const struct perf_event *event)
+{
+ return event->attr.config & ARM_CSPMU_EVENT_MASK;
+}
+
+static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event)
+{
+ return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT);
+}
+
+static u32 arm_cspmu_event_filter(const struct perf_event *event)
+{
+ return event->attr.config1 & ARM_CSPMU_FILTER_MASK;
+}
+
+static ssize_t arm_cspmu_identifier_show(struct device *dev,
+ struct device_attribute *attr,
+ char *page)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
+
+ return sysfs_emit(page, "%s\n", cspmu->identifier);
+}
+
+static struct device_attribute arm_cspmu_identifier_attr =
+ __ATTR(identifier, 0444, arm_cspmu_identifier_show, NULL);
+
+static struct attribute *arm_cspmu_identifier_attrs[] = {
+ &arm_cspmu_identifier_attr.attr,
+ NULL,
+};
+
+static struct attribute_group arm_cspmu_identifier_attr_group = {
+ .attrs = arm_cspmu_identifier_attrs,
+};
+
+static const char *arm_cspmu_get_identifier(const struct arm_cspmu *cspmu)
+{
+ const char *identifier =
+ devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",
+ cspmu->impl.pmiidr);
+ return identifier;
+}
+
+static const char *arm_cspmu_type_str[ACPI_APMT_NODE_TYPE_COUNT] = {
+ "mc",
+ "smmu",
+ "pcie",
+ "acpi",
+ "cache",
+};
+
+static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu)
+{
+ struct device *dev;
+ struct acpi_apmt_node *apmt_node;
+ u8 pmu_type;
+ char *name;
+ char acpi_hid_string[ACPI_ID_LEN] = { 0 };
+ static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 };
+
+ dev = cspmu->dev;
+ apmt_node = cspmu->apmt_node;
+ pmu_type = apmt_node->type;
+
+ if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) {
+ dev_err(dev, "unsupported PMU type-%u\n", pmu_type);
+ return NULL;
+ }
+
+ if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) {
+ memcpy(acpi_hid_string,
+ &apmt_node->inst_primary,
+ sizeof(apmt_node->inst_primary));
+ name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME,
+ arm_cspmu_type_str[pmu_type],
+ acpi_hid_string,
+ apmt_node->inst_secondary);
+ } else {
+ name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME,
+ arm_cspmu_type_str[pmu_type],
+ atomic_fetch_inc(&pmu_idx[pmu_type]));
+ }
+
+ return name;
+}
+
+static ssize_t arm_cspmu_cpumask_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pmu *pmu = dev_get_drvdata(dev);
+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
+ struct dev_ext_attribute *eattr =
+ container_of(attr, struct dev_ext_attribute, attr);
+ unsigned long mask_id = (unsigned long)eattr->var;
+ const cpumask_t *cpumask;
+
+ switch (mask_id) {
+ case ARM_CSPMU_ACTIVE_CPU_MASK:
+ cpumask = &cspmu->active_cpu;
+ break;
+ case ARM_CSPMU_ASSOCIATED_CPU_MASK:
+ cpumask = &cspmu->associated_cpus;
+ break;
+ default:
+ return 0;
+ }
+ return cpumap_print_to_pagebuf(true, buf, cpumask);
+}
+
+static struct attribute *arm_cspmu_cpumask_attrs[] = {
+ ARM_CSPMU_CPUMASK_ATTR(cpumask, ARM_CSPMU_ACTIVE_CPU_MASK),
+ ARM_CSPMU_CPUMASK_ATTR(associated_cpus, ARM_CSPMU_ASSOCIATED_CPU_MASK),
+ NULL,
+};
+
+static struct attribute_group arm_cspmu_cpumask_attr_group = {
+ .attrs = arm_cspmu_cpumask_attrs,
+};
+
+struct impl_match {
+ u32 pmiidr;
+ u32 mask;
+ int (*impl_init_ops)(struct arm_cspmu *cspmu);
+};
+
+static const struct impl_match impl_match[] = {
+ {}
+};
+
+static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)
+{
+ int ret;
+ struct acpi_apmt_node *apmt_node = cspmu->apmt_node;
+ struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
+ const struct impl_match *match = impl_match;
+
+ /*
+ * Get PMU implementer and product id from APMT node.
+ * If APMT node doesn't have implementer/product id, try get it
+ * from PMIIDR.
+ */
+ cspmu->impl.pmiidr =
+ (apmt_node->impl_id) ? apmt_node->impl_id :
+ readl(cspmu->base0 + PMIIDR);
+
+ /* Find implementer specific attribute ops. */
+ for (; match->pmiidr; match++) {
+ const u32 mask = match->mask;
+
+ if ((match->pmiidr & mask) == (cspmu->impl.pmiidr & mask)) {
+ ret = match->impl_init_ops(cspmu);
+ if (ret)
+ return ret;
+
+ break;
+ }
+ }
+
+ /* Use default callbacks if implementer doesn't provide one. */
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter);
+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible);
+
+ return 0;
+}
+
+static struct attribute_group *
+arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)
+{
+ struct attribute_group *event_group;
+ struct device *dev = cspmu->dev;
+ const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
+
+ event_group =
+ devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
+ if (!event_group)
+ return NULL;
+
+ event_group->name = "events";
+ event_group->is_visible = impl_ops->event_attr_is_visible;
+ event_group->attrs = impl_ops->get_event_attrs(cspmu);
+
+ if (!event_group->attrs)
+ return NULL;
+
+ return event_group;
+}
+
+static struct attribute_group *
+arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)
+{
+ struct attribute_group *format_group;
+ struct device *dev = cspmu->dev;
+
+ format_group =
+ devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
+ if (!format_group)
+ return NULL;
+
+ format_group->name = "format";
+ format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);
+
+ if (!format_group->attrs)
+ return NULL;
+
+ return format_group;
+}
+
+static struct attribute_group **
+arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)
+{
+ struct attribute_group **attr_groups = NULL;
+ struct device *dev = cspmu->dev;
+ const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
+ int ret;
+
+ ret = arm_cspmu_init_impl_ops(cspmu);
+ if (ret)
+ return NULL;
+
+ cspmu->identifier = impl_ops->get_identifier(cspmu);
+ cspmu->name = impl_ops->get_name(cspmu);
+
+ if (!cspmu->identifier || !cspmu->name)
+ return NULL;
+
+ attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *),
+ GFP_KERNEL);
+ if (!attr_groups)
+ return NULL;
+
+ attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);
+ attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);
+ attr_groups[2] = &arm_cspmu_identifier_attr_group;
+ attr_groups[3] = &arm_cspmu_cpumask_attr_group;
+
+ if (!attr_groups[0] || !attr_groups[1])
+ return NULL;
+
+ return attr_groups;
+}
+
+static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)
+{
+ u32 pmcr = 0;
+
+ pmcr |= PMCR_P;
+ pmcr |= PMCR_C;
+ writel(pmcr, cspmu->base0 + PMCR);
+}
+
+static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)
+{
+ writel(PMCR_E, cspmu->base0 + PMCR);
+}
+
+static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)
+{
+ writel(0, cspmu->base0 + PMCR);
+}
+
+static void arm_cspmu_enable(struct pmu *pmu)
+{
+ bool disabled;
+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
+
+ disabled = bitmap_empty(cspmu->hw_events.used_ctrs,
+ cspmu->num_logical_ctrs);
+
+ if (disabled)
+ return;
+
+ arm_cspmu_start_counters(cspmu);
+}
+
+static void arm_cspmu_disable(struct pmu *pmu)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
+
+ arm_cspmu_stop_counters(cspmu);
+}
+
+static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events,
+ struct perf_event *event)
+{
+ int idx;
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+
+ if (supports_cycle_counter(cspmu)) {
+ if (cspmu->impl.ops.is_cycle_counter_event(event)) {
+ /* Search for available cycle counter. */
+ if (test_and_set_bit(cspmu->cycle_counter_logical_idx,
+ hw_events->used_ctrs))
+ return -EAGAIN;
+
+ return cspmu->cycle_counter_logical_idx;
+ }
+
+ /*
+ * Search a regular counter from the used counter bitmap.
+ * The cycle counter divides the bitmap into two parts. Search
+ * the first then second half to exclude the cycle counter bit.
+ */
+ idx = find_first_zero_bit(hw_events->used_ctrs,
+ cspmu->cycle_counter_logical_idx);
+ if (idx >= cspmu->cycle_counter_logical_idx) {
+ idx = find_next_zero_bit(
+ hw_events->used_ctrs,
+ cspmu->num_logical_ctrs,
+ cspmu->cycle_counter_logical_idx + 1);
+ }
+ } else {
+ idx = find_first_zero_bit(hw_events->used_ctrs,
+ cspmu->num_logical_ctrs);
+ }
+
+ if (idx >= cspmu->num_logical_ctrs)
+ return -EAGAIN;
+
+ set_bit(idx, hw_events->used_ctrs);
+
+ return idx;
+}
+
+static bool arm_cspmu_validate_event(struct pmu *pmu,
+ struct arm_cspmu_hw_events *hw_events,
+ struct perf_event *event)
+{
+ if (is_software_event(event))
+ return true;
+
+ /* Reject groups spanning multiple HW PMUs. */
+ if (event->pmu != pmu)
+ return false;
+
+ return (arm_cspmu_get_event_idx(hw_events, event) >= 0);
+}
+
+/*
+ * Make sure the group of events can be scheduled at once
+ * on the PMU.
+ */
+static bool arm_cspmu_validate_group(struct perf_event *event)
+{
+ struct perf_event *sibling, *leader = event->group_leader;
+ struct arm_cspmu_hw_events fake_hw_events;
+
+ if (event->group_leader == event)
+ return true;
+
+ memset(&fake_hw_events, 0, sizeof(fake_hw_events));
+
+ if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events, leader))
+ return false;
+
+ for_each_sibling_event(sibling, leader) {
+ if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events,
+ sibling))
+ return false;
+ }
+
+ return arm_cspmu_validate_event(event->pmu, &fake_hw_events, event);
+}
+
+static int arm_cspmu_event_init(struct perf_event *event)
+{
+ struct arm_cspmu *cspmu;
+ struct hw_perf_event *hwc = &event->hw;
+
+ cspmu = to_arm_cspmu(event->pmu);
+
+ /*
+ * Following other "uncore" PMUs, we do not support sampling mode or
+ * attach to a task (per-process mode).
+ */
+ if (is_sampling_event(event)) {
+ dev_dbg(cspmu->pmu.dev,
+ "Can't support sampling events\n");
+ return -EOPNOTSUPP;
+ }
+
+ if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
+ dev_dbg(cspmu->pmu.dev,
+ "Can't support per-task counters\n");
+ return -EINVAL;
+ }
+
+ /*
+ * Make sure the CPU assignment is on one of the CPUs associated with
+ * this PMU.
+ */
+ if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {
+ dev_dbg(cspmu->pmu.dev,
+ "Requested cpu is not associated with the PMU\n");
+ return -EINVAL;
+ }
+
+ /* Enforce the current active CPU to handle the events in this PMU. */
+ event->cpu = cpumask_first(&cspmu->active_cpu);
+ if (event->cpu >= nr_cpu_ids)
+ return -EINVAL;
+
+ if (!arm_cspmu_validate_group(event))
+ return -EINVAL;
+
+ /*
+ * The logical counter id is tracked with hw_perf_event.extra_reg.idx.
+ * The physical counter id is tracked with hw_perf_event.idx.
+ * We don't assign an index until we actually place the event onto
+ * hardware. Use -1 to signify that we haven't decided where to put it
+ * yet.
+ */
+ hwc->idx = -1;
+ hwc->extra_reg.idx = -1;
+ hwc->config = cspmu->impl.ops.event_type(event);
+
+ return 0;
+}
+
+static inline u32 counter_offset(u32 reg_sz, u32 ctr_idx)
+{
+ return (PMEVCNTR_LO + (reg_sz * ctr_idx));
+}
+
+static void arm_cspmu_write_counter(struct perf_event *event, u64 val)
+{
+ u32 offset;
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+
+ if (use_64b_counter_reg(cspmu)) {
+ offset = counter_offset(sizeof(u64), event->hw.idx);
+
+ writeq(val, cspmu->base1 + offset);
+ } else {
+ offset = counter_offset(sizeof(u32), event->hw.idx);
+
+ writel(lower_32_bits(val), cspmu->base1 + offset);
+ }
+}
+
+static u64 arm_cspmu_read_counter(struct perf_event *event)
+{
+ u32 offset;
+ const void __iomem *counter_addr;
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+
+ if (use_64b_counter_reg(cspmu)) {
+ offset = counter_offset(sizeof(u64), event->hw.idx);
+ counter_addr = cspmu->base1 + offset;
+
+ return supports_64bit_atomics(cspmu) ?
+ readq(counter_addr) :
+ read_reg64_hilohi(counter_addr, HILOHI_MAX_POLL);
+ }
+
+ offset = counter_offset(sizeof(u32), event->hw.idx);
+ return readl(cspmu->base1 + offset);
+}
+
+/*
+ * arm_cspmu_set_event_period: Set the period for the counter.
+ *
+ * To handle cases of extreme interrupt latency, we program
+ * the counter with half of the max count for the counters.
+ */
+static void arm_cspmu_set_event_period(struct perf_event *event)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ u64 val = counter_mask(cspmu) >> 1ULL;
+
+ local64_set(&event->hw.prev_count, val);
+ arm_cspmu_write_counter(event, val);
+}
+
+static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)
+{
+ u32 reg_id, reg_bit, inten_off, cnten_off;
+
+ reg_id = COUNTER_TO_SET_CLR_ID(idx);
+ reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
+
+ inten_off = PMINTENSET + (4 * reg_id);
+ cnten_off = PMCNTENSET + (4 * reg_id);
+
+ writel(BIT(reg_bit), cspmu->base0 + inten_off);
+ writel(BIT(reg_bit), cspmu->base0 + cnten_off);
+}
+
+static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)
+{
+ u32 reg_id, reg_bit, inten_off, cnten_off;
+
+ reg_id = COUNTER_TO_SET_CLR_ID(idx);
+ reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
+
+ inten_off = PMINTENCLR + (4 * reg_id);
+ cnten_off = PMCNTENCLR + (4 * reg_id);
+
+ writel(BIT(reg_bit), cspmu->base0 + cnten_off);
+ writel(BIT(reg_bit), cspmu->base0 + inten_off);
+}
+
+static void arm_cspmu_event_update(struct perf_event *event)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u64 delta, prev, now;
+
+ do {
+ prev = local64_read(&hwc->prev_count);
+ now = arm_cspmu_read_counter(event);
+ } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
+
+ delta = (now - prev) & counter_mask(cspmu);
+ local64_add(delta, &event->count);
+}
+
+static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,
+ struct hw_perf_event *hwc)
+{
+ u32 offset = PMEVTYPER + (4 * hwc->idx);
+
+ writel(hwc->config, cspmu->base0 + offset);
+}
+
+static inline void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
+ struct hw_perf_event *hwc,
+ u32 filter)
+{
+ u32 offset = PMEVFILTR + (4 * hwc->idx);
+
+ writel(filter, cspmu->base0 + offset);
+}
+
+static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)
+{
+ u32 offset = PMCCFILTR;
+
+ writel(filter, cspmu->base0 + offset);
+}
+
+static void arm_cspmu_start(struct perf_event *event, int pmu_flags)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u32 filter;
+
+ /* We always reprogram the counter */
+ if (pmu_flags & PERF_EF_RELOAD)
+ WARN_ON(!(hwc->state & PERF_HES_UPTODATE));
+
+ arm_cspmu_set_event_period(event);
+
+ filter = cspmu->impl.ops.event_filter(event);
+
+ if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {
+ arm_cspmu_set_cc_filter(cspmu, filter);
+ } else {
+ arm_cspmu_set_event(cspmu, hwc);
+ arm_cspmu_set_ev_filter(cspmu, hwc, filter);
+ }
+
+ hwc->state = 0;
+
+ arm_cspmu_enable_counter(cspmu, hwc->idx);
+}
+
+static void arm_cspmu_stop(struct perf_event *event, int pmu_flags)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (hwc->state & PERF_HES_STOPPED)
+ return;
+
+ arm_cspmu_disable_counter(cspmu, hwc->idx);
+ arm_cspmu_event_update(event);
+
+ hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+}
+
+static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)
+{
+ return (idx == cspmu->cycle_counter_logical_idx) ?
+ ARM_CSPMU_CYCLE_CNTR_IDX : idx;
+}
+
+static int arm_cspmu_add(struct perf_event *event, int flags)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
+ struct hw_perf_event *hwc = &event->hw;
+ int idx;
+
+ if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
+ &cspmu->associated_cpus)))
+ return -ENOENT;
+
+ idx = arm_cspmu_get_event_idx(hw_events, event);
+ if (idx < 0)
+ return idx;
+
+ hw_events->events[idx] = event;
+ hwc->idx = to_phys_idx(cspmu, idx);
+ hwc->extra_reg.idx = idx;
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+ if (flags & PERF_EF_START)
+ arm_cspmu_start(event, PERF_EF_RELOAD);
+
+ /* Propagate changes to the userspace mapping. */
+ perf_event_update_userpage(event);
+
+ return 0;
+}
+
+static void arm_cspmu_del(struct perf_event *event, int flags)
+{
+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
+ struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->extra_reg.idx;
+
+ arm_cspmu_stop(event, PERF_EF_UPDATE);
+
+ hw_events->events[idx] = NULL;
+
+ clear_bit(idx, hw_events->used_ctrs);
+
+ perf_event_update_userpage(event);
+}
+
+static void arm_cspmu_read(struct perf_event *event)
+{
+ arm_cspmu_event_update(event);
+}
+
+static struct arm_cspmu *arm_cspmu_alloc(struct platform_device *pdev)
+{
+ struct acpi_apmt_node *apmt_node;
+ struct arm_cspmu *cspmu;
+ struct device *dev;
+
+ dev = &pdev->dev;
+ apmt_node = *(struct acpi_apmt_node **)dev_get_platdata(dev);
+ if (!apmt_node) {
+ dev_err(dev, "failed to get APMT node\n");
+ return NULL;
+ }
+
+ cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);
+ if (!cspmu)
+ return NULL;
+
+ cspmu->dev = dev;
+ cspmu->apmt_node = apmt_node;
+
+ platform_set_drvdata(pdev, cspmu);
+
+ return cspmu;
+}
+
+static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)
+{
+ struct device *dev;
+ struct platform_device *pdev;
+ struct acpi_apmt_node *apmt_node;
+
+ dev = cspmu->dev;
+ pdev = to_platform_device(dev);
+ apmt_node = cspmu->apmt_node;
+
+ /* Base address for page 0. */
+ cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(cspmu->base0)) {
+ dev_err(dev, "ioremap failed for page-0 resource\n");
+ return PTR_ERR(cspmu->base0);
+ }
+
+ /* Base address for page 1 if supported. Otherwise point to page 0. */
+ cspmu->base1 = cspmu->base0;
+ if (CHECK_APMT_FLAG(apmt_node->flags, DUAL_PAGE, SUPP)) {
+ cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);
+ if (IS_ERR(cspmu->base1)) {
+ dev_err(dev, "ioremap failed for page-1 resource\n");
+ return PTR_ERR(cspmu->base1);
+ }
+ }
+
+ cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);
+
+ cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;
+
+ cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;
+
+ if (supports_cycle_counter(cspmu)) {
+ /*
+ * The last logical counter is mapped to cycle counter if
+ * there is a gap between regular and cycle counter. Otherwise,
+ * logical and physical have 1-to-1 mapping.
+ */
+ cspmu->cycle_counter_logical_idx =
+ (cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?
+ cspmu->num_logical_ctrs - 1 :
+ ARM_CSPMU_CYCLE_CNTR_IDX;
+ }
+
+ cspmu->num_set_clr_reg =
+ DIV_ROUND_UP(cspmu->num_logical_ctrs,
+ ARM_CSPMU_SET_CLR_COUNTER_NUM);
+
+ cspmu->hw_events.events =
+ devm_kcalloc(dev, cspmu->num_logical_ctrs,
+ sizeof(*cspmu->hw_events.events), GFP_KERNEL);
+
+ if (!cspmu->hw_events.events)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,
+ u32 *pmovs)
+{
+ int i;
+ u32 pmovclr_offset = PMOVSCLR;
+ u32 has_overflowed = 0;
+
+ for (i = 0; i < cspmu->num_set_clr_reg; ++i) {
+ pmovs[i] = readl(cspmu->base1 + pmovclr_offset);
+ has_overflowed |= pmovs[i];
+ writel(pmovs[i], cspmu->base1 + pmovclr_offset);
+ pmovclr_offset += sizeof(u32);
+ }
+
+ return has_overflowed != 0;
+}
+
+static irqreturn_t arm_cspmu_handle_irq(int irq_num, void *dev)
+{
+ int idx, has_overflowed;
+ struct perf_event *event;
+ struct arm_cspmu *cspmu = dev;
+ DECLARE_BITMAP(pmovs, ARM_CSPMU_MAX_HW_CNTRS);
+ bool handled = false;
+
+ arm_cspmu_stop_counters(cspmu);
+
+ has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);
+ if (!has_overflowed)
+ goto done;
+
+ for_each_set_bit(idx, cspmu->hw_events.used_ctrs,
+ cspmu->num_logical_ctrs) {
+ event = cspmu->hw_events.events[idx];
+
+ if (!event)
+ continue;
+
+ if (!test_bit(event->hw.idx, pmovs))
+ continue;
+
+ arm_cspmu_event_update(event);
+ arm_cspmu_set_event_period(event);
+
+ handled = true;
+ }
+
+done:
+ arm_cspmu_start_counters(cspmu);
+ return IRQ_RETVAL(handled);
+}
+
+static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
+{
+ int irq, ret;
+ struct device *dev;
+ struct platform_device *pdev;
+ struct acpi_apmt_node *apmt_node;
+
+ dev = cspmu->dev;
+ pdev = to_platform_device(dev);
+ apmt_node = cspmu->apmt_node;
+
+ /* Skip IRQ request if the PMU does not support overflow interrupt. */
+ if (apmt_node->ovflw_irq == 0)
+ return 0;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0)
+ return irq;
+
+ ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq,
+ IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev),
+ cspmu);
+ if (ret) {
+ dev_err(dev, "Could not request IRQ %d\n", irq);
+ return ret;
+ }
+
+ cspmu->irq = irq;
+
+ return 0;
+}
+
+static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
+{
+ u32 acpi_uid;
+ struct device *cpu_dev = get_cpu_device(cpu);
+ struct acpi_device *acpi_dev = ACPI_COMPANION(cpu_dev);
+
+ if (!cpu_dev)
+ return -ENODEV;
+
+ while (acpi_dev) {
+ if (!strcmp(acpi_device_hid(acpi_dev),
+ ACPI_PROCESSOR_CONTAINER_HID) &&
+ !kstrtouint(acpi_device_uid(acpi_dev), 0, &acpi_uid) &&
+ acpi_uid == container_uid)
+ return 0;
+
+ acpi_dev = acpi_dev_parent(acpi_dev);
+ }
+
+ return -ENODEV;
+}
+
+static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)
+{
+ struct device *dev;
+ struct acpi_apmt_node *apmt_node;
+ int affinity_flag;
+ int cpu;
+
+ dev = cspmu->pmu.dev;
+ apmt_node = cspmu->apmt_node;
+ affinity_flag = apmt_node->flags & ACPI_APMT_FLAGS_AFFINITY;
+
+ if (affinity_flag == ACPI_APMT_FLAGS_AFFINITY_PROC) {
+ for_each_possible_cpu(cpu) {
+ if (apmt_node->proc_affinity ==
+ get_acpi_id_for_cpu(cpu)) {
+ cpumask_set_cpu(cpu, &cspmu->associated_cpus);
+ break;
+ }
+ }
+ } else {
+ for_each_possible_cpu(cpu) {
+ if (arm_cspmu_find_cpu_container(
+ cpu, apmt_node->proc_affinity))
+ continue;
+
+ cpumask_set_cpu(cpu, &cspmu->associated_cpus);
+ }
+ }
+
+ if (cpumask_empty(&cspmu->associated_cpus)) {
+ dev_dbg(dev, "No cpu associated with the PMU\n");
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)
+{
+ int ret, capabilities;
+ struct attribute_group **attr_groups;
+
+ attr_groups = arm_cspmu_alloc_attr_group(cspmu);
+ if (!attr_groups)
+ return -ENOMEM;
+
+ ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state,
+ &cspmu->cpuhp_node);
+ if (ret)
+ return ret;
+
+ capabilities = PERF_PMU_CAP_NO_EXCLUDE;
+ if (cspmu->irq == 0)
+ capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
+
+ cspmu->pmu = (struct pmu){
+ .task_ctx_nr = perf_invalid_context,
+ .module = THIS_MODULE,
+ .pmu_enable = arm_cspmu_enable,
+ .pmu_disable = arm_cspmu_disable,
+ .event_init = arm_cspmu_event_init,
+ .add = arm_cspmu_add,
+ .del = arm_cspmu_del,
+ .start = arm_cspmu_start,
+ .stop = arm_cspmu_stop,
+ .read = arm_cspmu_read,
+ .attr_groups = (const struct attribute_group **)attr_groups,
+ .capabilities = capabilities,
+ };
+
+ /* Hardware counter init */
+ arm_cspmu_stop_counters(cspmu);
+ arm_cspmu_reset_counters(cspmu);
+
+ ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);
+ if (ret) {
+ cpuhp_state_remove_instance(arm_cspmu_cpuhp_state,
+ &cspmu->cpuhp_node);
+ }
+
+ return ret;
+}
+
+static int arm_cspmu_device_probe(struct platform_device *pdev)
+{
+ int ret;
+ struct arm_cspmu *cspmu;
+
+ cspmu = arm_cspmu_alloc(pdev);
+ if (!cspmu)
+ return -ENOMEM;
+
+ ret = arm_cspmu_init_mmio(cspmu);
+ if (ret)
+ return ret;
+
+ ret = arm_cspmu_request_irq(cspmu);
+ if (ret)
+ return ret;
+
+ ret = arm_cspmu_get_cpus(cspmu);
+ if (ret)
+ return ret;
+
+ ret = arm_cspmu_register_pmu(cspmu);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static int arm_cspmu_device_remove(struct platform_device *pdev)
+{
+ struct arm_cspmu *cspmu = platform_get_drvdata(pdev);
+
+ perf_pmu_unregister(&cspmu->pmu);
+ cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);
+
+ return 0;
+}
+
+static struct platform_driver arm_cspmu_driver = {
+ .driver = {
+ .name = DRVNAME,
+ .suppress_bind_attrs = true,
+ },
+ .probe = arm_cspmu_device_probe,
+ .remove = arm_cspmu_device_remove,
+};
+
+static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)
+{
+ cpumask_set_cpu(cpu, &cspmu->active_cpu);
+ WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));
+}
+
+static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node)
+{
+ struct arm_cspmu *cspmu =
+ hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
+
+ if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))
+ return 0;
+
+ /* If the PMU is already managed, there is nothing to do */
+ if (!cpumask_empty(&cspmu->active_cpu))
+ return 0;
+
+ /* Use this CPU for event counting */
+ arm_cspmu_set_active_cpu(cpu, cspmu);
+
+ return 0;
+}
+
+static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
+{
+ int dst;
+ struct cpumask online_supported;
+
+ struct arm_cspmu *cspmu =
+ hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
+
+ /* Nothing to do if this CPU doesn't own the PMU */
+ if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))
+ return 0;
+
+ /* Choose a new CPU to migrate ownership of the PMU to */
+ cpumask_and(&online_supported, &cspmu->associated_cpus,
+ cpu_online_mask);
+ dst = cpumask_any_but(&online_supported, cpu);
+ if (dst >= nr_cpu_ids)
+ return 0;
+
+ /* Use this CPU for event counting */
+ perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);
+ arm_cspmu_set_active_cpu(dst, cspmu);
+
+ return 0;
+}
+
+static int __init arm_cspmu_init(void)
+{
+ int ret;
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+ "perf/arm/cspmu:online",
+ arm_cspmu_cpu_online,
+ arm_cspmu_cpu_teardown);
+ if (ret < 0)
+ return ret;
+ arm_cspmu_cpuhp_state = ret;
+ return platform_driver_register(&arm_cspmu_driver);
+}
+
+static void __exit arm_cspmu_exit(void)
+{
+ platform_driver_unregister(&arm_cspmu_driver);
+ cpuhp_remove_multi_state(arm_cspmu_cpuhp_state);
+}
+
+module_init(arm_cspmu_init);
+module_exit(arm_cspmu_exit);
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h
new file mode 100644
index 000000000000..51323b175a4a
--- /dev/null
+++ b/drivers/perf/arm_cspmu/arm_cspmu.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * ARM CoreSight Architecture PMU driver.
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ */
+
+#ifndef __ARM_CSPMU_H__
+#define __ARM_CSPMU_H__
+
+#include <linux/acpi.h>
+#include <linux/bitfield.h>
+#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#define to_arm_cspmu(p) (container_of(p, struct arm_cspmu, pmu))
+
+#define ARM_CSPMU_EXT_ATTR(_name, _func, _config) \
+ (&((struct dev_ext_attribute[]){ \
+ { \
+ .attr = __ATTR(_name, 0444, _func, NULL), \
+ .var = (void *)_config \
+ } \
+ })[0].attr.attr)
+
+#define ARM_CSPMU_FORMAT_ATTR(_name, _config) \
+ ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_sysfs_format_show, (char *)_config)
+
+#define ARM_CSPMU_EVENT_ATTR(_name, _config) \
+ PMU_EVENT_ATTR_ID(_name, arm_cspmu_sysfs_event_show, _config)
+
+
+/* Default event id mask */
+#define ARM_CSPMU_EVENT_MASK GENMASK_ULL(63, 0)
+
+/* Default filter value mask */
+#define ARM_CSPMU_FILTER_MASK GENMASK_ULL(63, 0)
+
+/* Default event format */
+#define ARM_CSPMU_FORMAT_EVENT_ATTR \
+ ARM_CSPMU_FORMAT_ATTR(event, "config:0-32")
+
+/* Default filter format */
+#define ARM_CSPMU_FORMAT_FILTER_ATTR \
+ ARM_CSPMU_FORMAT_ATTR(filter, "config1:0-31")
+
+/*
+ * This is the default event number for cycle count, if supported, since the
+ * ARM Coresight PMU specification does not define a standard event code
+ * for cycle count.
+ */
+#define ARM_CSPMU_EVT_CYCLES_DEFAULT (0x1ULL << 32)
+
+/*
+ * The ARM Coresight PMU supports up to 256 event counters.
+ * If the counters are larger-than 32-bits, then the PMU includes at
+ * most 128 counters.
+ */
+#define ARM_CSPMU_MAX_HW_CNTRS 256
+
+/* The cycle counter, if implemented, is located at counter[31]. */
+#define ARM_CSPMU_CYCLE_CNTR_IDX 31
+
+/* PMIIDR register field */
+#define ARM_CSPMU_PMIIDR_IMPLEMENTER GENMASK(11, 0)
+#define ARM_CSPMU_PMIIDR_PRODUCTID GENMASK(31, 20)
+
+struct arm_cspmu;
+
+/* This tracks the events assigned to each counter in the PMU. */
+struct arm_cspmu_hw_events {
+ /* The events that are active on the PMU for a given logical index. */
+ struct perf_event **events;
+
+ /*
+ * Each bit indicates a logical counter is being used (or not) for an
+ * event. If cycle counter is supported and there is a gap between
+ * regular and cycle counter, the last logical counter is mapped to
+ * cycle counter. Otherwise, logical and physical have 1-to-1 mapping.
+ */
+ DECLARE_BITMAP(used_ctrs, ARM_CSPMU_MAX_HW_CNTRS);
+};
+
+/* Contains ops to query vendor/implementer specific attribute. */
+struct arm_cspmu_impl_ops {
+ /* Get event attributes */
+ struct attribute **(*get_event_attrs)(const struct arm_cspmu *cspmu);
+ /* Get format attributes */
+ struct attribute **(*get_format_attrs)(const struct arm_cspmu *cspmu);
+ /* Get string identifier */
+ const char *(*get_identifier)(const struct arm_cspmu *cspmu);
+ /* Get PMU name to register to core perf */
+ const char *(*get_name)(const struct arm_cspmu *cspmu);
+ /* Check if the event corresponds to cycle count event */
+ bool (*is_cycle_counter_event)(const struct perf_event *event);
+ /* Decode event type/id from configs */
+ u32 (*event_type)(const struct perf_event *event);
+ /* Decode filter value from configs */
+ u32 (*event_filter)(const struct perf_event *event);
+ /* Hide/show unsupported events */
+ umode_t (*event_attr_is_visible)(struct kobject *kobj,
+ struct attribute *attr, int unused);
+};
+
+/* Vendor/implementer descriptor. */
+struct arm_cspmu_impl {
+ u32 pmiidr;
+ struct arm_cspmu_impl_ops ops;
+ void *ctx;
+};
+
+/* Coresight PMU descriptor. */
+struct arm_cspmu {
+ struct pmu pmu;
+ struct device *dev;
+ struct acpi_apmt_node *apmt_node;
+ const char *name;
+ const char *identifier;
+ void __iomem *base0;
+ void __iomem *base1;
+ int irq;
+ cpumask_t associated_cpus;
+ cpumask_t active_cpu;
+ struct hlist_node cpuhp_node;
+
+ u32 pmcfgr;
+ u32 num_logical_ctrs;
+ u32 num_set_clr_reg;
+ int cycle_counter_logical_idx;
+
+ struct arm_cspmu_hw_events hw_events;
+
+ struct arm_cspmu_impl impl;
+};
+
+/* Default function to show event attribute in sysfs. */
+ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf);
+
+/* Default function to show format attribute in sysfs. */
+ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf);
+
+#endif /* __ARM_CSPMU_H__ */
From 84481be7167eba1957c8718c8b044a47965fdecb Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Fri, 11 Nov 2022 16:23:29 -0600
Subject: [PATCH 057/151] perf: arm_cspmu: Add support for NVIDIA SCF and MCF
attribute
Add support for NVIDIA System Cache Fabric (SCF) and Memory Control
Fabric (MCF) PMU attributes for CoreSight PMU implementation in
NVIDIA devices.
Acked-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Link: https://lore.kernel.org/r/20221111222330.48602-3-bwicaksono@nvidia.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/admin-guide/perf/index.rst | 1 +
Documentation/admin-guide/perf/nvidia-pmu.rst | 299 +++++++++++++
drivers/perf/arm_cspmu/Makefile | 3 +-
drivers/perf/arm_cspmu/arm_cspmu.c | 9 +
drivers/perf/arm_cspmu/nvidia_cspmu.c | 398 ++++++++++++++++++
drivers/perf/arm_cspmu/nvidia_cspmu.h | 17 +
6 files changed, 726 insertions(+), 1 deletion(-)
create mode 100644 Documentation/admin-guide/perf/nvidia-pmu.rst
create mode 100644 drivers/perf/arm_cspmu/nvidia_cspmu.c
create mode 100644 drivers/perf/arm_cspmu/nvidia_cspmu.h
diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst
index 793e1970bc05..7ed239408138 100644
--- a/Documentation/admin-guide/perf/index.rst
+++ b/Documentation/admin-guide/perf/index.rst
@@ -19,3 +19,4 @@ Performance monitor support
arm_dsu_pmu
thunderx2-pmu
alibaba_pmu
+ nvidia-pmu
diff --git a/Documentation/admin-guide/perf/nvidia-pmu.rst b/Documentation/admin-guide/perf/nvidia-pmu.rst
new file mode 100644
index 000000000000..2e0d47cfe7ea
--- /dev/null
+++ b/Documentation/admin-guide/perf/nvidia-pmu.rst
@@ -0,0 +1,299 @@
+=========================================================
+NVIDIA Tegra SoC Uncore Performance Monitoring Unit (PMU)
+=========================================================
+
+The NVIDIA Tegra SoC includes various system PMUs to measure key performance
+metrics like memory bandwidth, latency, and utilization:
+
+* Scalable Coherency Fabric (SCF)
+* NVLink-C2C0
+* NVLink-C2C1
+* CNVLink
+* PCIE
+
+PMU Driver
+----------
+
+The PMUs in this document are based on ARM CoreSight PMU Architecture as
+described in document: ARM IHI 0091. Since this is a standard architecture, the
+PMUs are managed by a common driver "arm-cs-arch-pmu". This driver describes
+the available events and configuration of each PMU in sysfs. Please see the
+sections below to get the sysfs path of each PMU. Like other uncore PMU drivers,
+the driver provides "cpumask" sysfs attribute to show the CPU id used to handle
+the PMU event. There is also "associated_cpus" sysfs attribute, which contains a
+list of CPUs associated with the PMU instance.
+
+.. _SCF_PMU_Section:
+
+SCF PMU
+-------
+
+The SCF PMU monitors system level cache events, CPU traffic, and
+strongly-ordered (SO) PCIE write traffic to local/remote memory. Please see
+:ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about the PMU
+traffic coverage.
+
+The events and configuration options of this PMU device are described in sysfs,
+see /sys/bus/event_sources/devices/nvidia_scf_pmu_<socket-id>.
+
+Example usage:
+
+* Count event id 0x0 in socket 0::
+
+ perf stat -a -e nvidia_scf_pmu_0/event=0x0/
+
+* Count event id 0x0 in socket 1::
+
+ perf stat -a -e nvidia_scf_pmu_1/event=0x0/
+
+NVLink-C2C0 PMU
+--------------------
+
+The NVLink-C2C0 PMU monitors incoming traffic from a GPU/CPU connected with
+NVLink-C2C (Chip-2-Chip) interconnect. The type of traffic captured by this PMU
+varies dependent on the chip configuration:
+
+* NVIDIA Grace Hopper Superchip: Hopper GPU is connected with Grace SoC.
+
+ In this config, the PMU captures GPU ATS translated or EGM traffic from the GPU.
+
+* NVIDIA Grace CPU Superchip: two Grace CPU SoCs are connected.
+
+ In this config, the PMU captures read and relaxed ordered (RO) writes from
+ PCIE device of the remote SoC.
+
+Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about
+the PMU traffic coverage.
+
+The events and configuration options of this PMU device are described in sysfs,
+see /sys/bus/event_sources/devices/nvidia_nvlink_c2c0_pmu_<socket-id>.
+
+Example usage:
+
+* Count event id 0x0 from the GPU/CPU connected with socket 0::
+
+ perf stat -a -e nvidia_nvlink_c2c0_pmu_0/event=0x0/
+
+* Count event id 0x0 from the GPU/CPU connected with socket 1::
+
+ perf stat -a -e nvidia_nvlink_c2c0_pmu_1/event=0x0/
+
+* Count event id 0x0 from the GPU/CPU connected with socket 2::
+
+ perf stat -a -e nvidia_nvlink_c2c0_pmu_2/event=0x0/
+
+* Count event id 0x0 from the GPU/CPU connected with socket 3::
+
+ perf stat -a -e nvidia_nvlink_c2c0_pmu_3/event=0x0/
+
+NVLink-C2C1 PMU
+-------------------
+
+The NVLink-C2C1 PMU monitors incoming traffic from a GPU connected with
+NVLink-C2C (Chip-2-Chip) interconnect. This PMU captures untranslated GPU
+traffic, in contrast with NvLink-C2C0 PMU that captures ATS translated traffic.
+Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about
+the PMU traffic coverage.
+
+The events and configuration options of this PMU device are described in sysfs,
+see /sys/bus/event_sources/devices/nvidia_nvlink_c2c1_pmu_<socket-id>.
+
+Example usage:
+
+* Count event id 0x0 from the GPU connected with socket 0::
+
+ perf stat -a -e nvidia_nvlink_c2c1_pmu_0/event=0x0/
+
+* Count event id 0x0 from the GPU connected with socket 1::
+
+ perf stat -a -e nvidia_nvlink_c2c1_pmu_1/event=0x0/
+
+* Count event id 0x0 from the GPU connected with socket 2::
+
+ perf stat -a -e nvidia_nvlink_c2c1_pmu_2/event=0x0/
+
+* Count event id 0x0 from the GPU connected with socket 3::
+
+ perf stat -a -e nvidia_nvlink_c2c1_pmu_3/event=0x0/
+
+CNVLink PMU
+---------------
+
+The CNVLink PMU monitors traffic from GPU and PCIE device on remote sockets
+to local memory. For PCIE traffic, this PMU captures read and relaxed ordered
+(RO) write traffic. Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section`
+for more info about the PMU traffic coverage.
+
+The events and configuration options of this PMU device are described in sysfs,
+see /sys/bus/event_sources/devices/nvidia_cnvlink_pmu_<socket-id>.
+
+Each SoC socket can be connected to one or more sockets via CNVLink. The user can
+use "rem_socket" bitmap parameter to select the remote socket(s) to monitor.
+Each bit represents the socket number, e.g. "rem_socket=0xE" corresponds to
+socket 1 to 3.
+/sys/bus/event_sources/devices/nvidia_cnvlink_pmu_<socket-id>/format/rem_socket
+shows the valid bits that can be set in the "rem_socket" parameter.
+
+The PMU can not distinguish the remote traffic initiator, therefore it does not
+provide filter to select the traffic source to monitor. It reports combined
+traffic from remote GPU and PCIE devices.
+
+Example usage:
+
+* Count event id 0x0 for the traffic from remote socket 1, 2, and 3 to socket 0::
+
+ perf stat -a -e nvidia_cnvlink_pmu_0/event=0x0,rem_socket=0xE/
+
+* Count event id 0x0 for the traffic from remote socket 0, 2, and 3 to socket 1::
+
+ perf stat -a -e nvidia_cnvlink_pmu_1/event=0x0,rem_socket=0xD/
+
+* Count event id 0x0 for the traffic from remote socket 0, 1, and 3 to socket 2::
+
+ perf stat -a -e nvidia_cnvlink_pmu_2/event=0x0,rem_socket=0xB/
+
+* Count event id 0x0 for the traffic from remote socket 0, 1, and 2 to socket 3::
+
+ perf stat -a -e nvidia_cnvlink_pmu_3/event=0x0,rem_socket=0x7/
+
+
+PCIE PMU
+------------
+
+The PCIE PMU monitors all read/write traffic from PCIE root ports to
+local/remote memory. Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section`
+for more info about the PMU traffic coverage.
+
+The events and configuration options of this PMU device are described in sysfs,
+see /sys/bus/event_sources/devices/nvidia_pcie_pmu_<socket-id>.
+
+Each SoC socket can support multiple root ports. The user can use
+"root_port" bitmap parameter to select the port(s) to monitor, i.e.
+"root_port=0xF" corresponds to root port 0 to 3.
+/sys/bus/event_sources/devices/nvidia_pcie_pmu_<socket-id>/format/root_port
+shows the valid bits that can be set in the "root_port" parameter.
+
+Example usage:
+
+* Count event id 0x0 from root port 0 and 1 of socket 0::
+
+ perf stat -a -e nvidia_pcie_pmu_0/event=0x0,root_port=0x3/
+
+* Count event id 0x0 from root port 0 and 1 of socket 1::
+
+ perf stat -a -e nvidia_pcie_pmu_1/event=0x0,root_port=0x3/
+
+.. _NVIDIA_Uncore_PMU_Traffic_Coverage_Section:
+
+Traffic Coverage
+----------------
+
+The PMU traffic coverage may vary dependent on the chip configuration:
+
+* **NVIDIA Grace Hopper Superchip**: Hopper GPU is connected with Grace SoC.
+
+ Example configuration with two Grace SoCs::
+
+ ********************************* *********************************
+ * SOCKET-A * * SOCKET-B *
+ * * * *
+ * :::::::: * * :::::::: *
+ * : PCIE : * * : PCIE : *
+ * :::::::: * * :::::::: *
+ * | * * | *
+ * | * * | *
+ * ::::::: ::::::::: * * ::::::::: ::::::: *
+ * : : : : * * : : : : *
+ * : GPU :<--NVLink-->: Grace :<---CNVLink--->: Grace :<--NVLink-->: GPU : *
+ * : : C2C : SoC : * * : SoC : C2C : : *
+ * ::::::: ::::::::: * * ::::::::: ::::::: *
+ * | | * * | | *
+ * | | * * | | *
+ * &&&&&&&& &&&&&&&& * * &&&&&&&& &&&&&&&& *
+ * & GMEM & & CMEM & * * & CMEM & & GMEM & *
+ * &&&&&&&& &&&&&&&& * * &&&&&&&& &&&&&&&& *
+ * * * *
+ ********************************* *********************************
+
+ GMEM = GPU Memory (e.g. HBM)
+ CMEM = CPU Memory (e.g. LPDDR5X)
+
+ |
+ | Following table contains traffic coverage of Grace SoC PMU in socket-A:
+
+ ::
+
+ +--------------+-------+-----------+-----------+-----+----------+----------+
+ | | Source |
+ + +-------+-----------+-----------+-----+----------+----------+
+ | Destination | |GPU ATS |GPU Not-ATS| | Socket-B | Socket-B |
+ | |PCI R/W|Translated,|Translated | CPU | CPU/PCIE1| GPU/PCIE2|
+ | | |EGM | | | | |
+ +==============+=======+===========+===========+=====+==========+==========+
+ | Local | PCIE |NVLink-C2C0|NVLink-C2C1| SCF | SCF PMU | CNVLink |
+ | SYSRAM/CMEM | PMU |PMU |PMU | PMU | | PMU |
+ +--------------+-------+-----------+-----------+-----+----------+----------+
+ | Local GMEM | PCIE | N/A |NVLink-C2C1| SCF | SCF PMU | CNVLink |
+ | | PMU | |PMU | PMU | | PMU |
+ +--------------+-------+-----------+-----------+-----+----------+----------+
+ | Remote | PCIE |NVLink-C2C0|NVLink-C2C1| SCF | | |
+ | SYSRAM/CMEM | PMU |PMU |PMU | PMU | N/A | N/A |
+ | over CNVLink | | | | | | |
+ +--------------+-------+-----------+-----------+-----+----------+----------+
+ | Remote GMEM | PCIE |NVLink-C2C0|NVLink-C2C1| SCF | | |
+ | over CNVLink | PMU |PMU |PMU | PMU | N/A | N/A |
+ +--------------+-------+-----------+-----------+-----+----------+----------+
+
+ PCIE1 traffic represents strongly ordered (SO) writes.
+ PCIE2 traffic represents reads and relaxed ordered (RO) writes.
+
+* **NVIDIA Grace CPU Superchip**: two Grace CPU SoCs are connected.
+
+ Example configuration with two Grace SoCs::
+
+ ******************* *******************
+ * SOCKET-A * * SOCKET-B *
+ * * * *
+ * :::::::: * * :::::::: *
+ * : PCIE : * * : PCIE : *
+ * :::::::: * * :::::::: *
+ * | * * | *
+ * | * * | *
+ * ::::::::: * * ::::::::: *
+ * : : * * : : *
+ * : Grace :<--------NVLink------->: Grace : *
+ * : SoC : * C2C * : SoC : *
+ * ::::::::: * * ::::::::: *
+ * | * * | *
+ * | * * | *
+ * &&&&&&&& * * &&&&&&&& *
+ * & CMEM & * * & CMEM & *
+ * &&&&&&&& * * &&&&&&&& *
+ * * * *
+ ******************* *******************
+
+ GMEM = GPU Memory (e.g. HBM)
+ CMEM = CPU Memory (e.g. LPDDR5X)
+
+ |
+ | Following table contains traffic coverage of Grace SoC PMU in socket-A:
+
+ ::
+
+ +-----------------+-----------+---------+----------+-------------+
+ | | Source |
+ + +-----------+---------+----------+-------------+
+ | Destination | | | Socket-B | Socket-B |
+ | | PCI R/W | CPU | CPU/PCIE1| PCIE2 |
+ | | | | | |
+ +=================+===========+=========+==========+=============+
+ | Local | PCIE PMU | SCF PMU | SCF PMU | NVLink-C2C0 |
+ | SYSRAM/CMEM | | | | PMU |
+ +-----------------+-----------+---------+----------+-------------+
+ | Remote | | | | |
+ | SYSRAM/CMEM | PCIE PMU | SCF PMU | N/A | N/A |
+ | over NVLink-C2C | | | | |
+ +-----------------+-----------+---------+----------+-------------+
+
+ PCIE1 traffic represents strongly ordered (SO) writes.
+ PCIE2 traffic represents reads and relaxed ordered (RO) writes.
diff --git a/drivers/perf/arm_cspmu/Makefile b/drivers/perf/arm_cspmu/Makefile
index 9bc76de4002d..641db85c018b 100644
--- a/drivers/perf/arm_cspmu/Makefile
+++ b/drivers/perf/arm_cspmu/Makefile
@@ -3,4 +3,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += \
- arm_cspmu.o
+ arm_cspmu.o \
+ nvidia_cspmu.o
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c
index e8f57aa9f047..e851eeb33f4a 100644
--- a/drivers/perf/arm_cspmu/arm_cspmu.c
+++ b/drivers/perf/arm_cspmu/arm_cspmu.c
@@ -31,6 +31,7 @@
#include <acpi/processor.h>
#include "arm_cspmu.h"
+#include "nvidia_cspmu.h"
#define PMUNAME "arm_cspmu"
#define DRVNAME "arm-cs-arch-pmu"
@@ -116,6 +117,9 @@
*/
#define HILOHI_MAX_POLL 1000
+/* JEDEC-assigned JEP106 identification code */
+#define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B
+
static unsigned long arm_cspmu_cpuhp_state;
/*
@@ -382,6 +386,11 @@ struct impl_match {
};
static const struct impl_match impl_match[] = {
+ {
+ .pmiidr = ARM_CSPMU_IMPL_ID_NVIDIA,
+ .mask = ARM_CSPMU_PMIIDR_IMPLEMENTER,
+ .impl_init_ops = nv_cspmu_init_ops
+ },
{}
};
diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c
new file mode 100644
index 000000000000..c795414ec7c7
--- /dev/null
+++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c
@@ -0,0 +1,398 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ */
+
+/* Support for NVIDIA specific attributes. */
+
+#include <linux/topology.h>
+
+#include "nvidia_cspmu.h"
+
+#define NV_PCIE_PORT_COUNT 10ULL
+#define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
+
+#define NV_NVL_C2C_PORT_COUNT 2ULL
+#define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
+
+#define NV_CNVL_PORT_COUNT 4ULL
+#define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
+
+#define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
+
+#define NV_PRODID_MASK GENMASK(31, 0)
+
+#define NV_FORMAT_NAME_GENERIC 0
+
+#define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
+
+#define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
+ ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
+
+#define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
+
+struct nv_cspmu_ctx {
+ const char *name;
+ u32 filter_mask;
+ u32 filter_default_val;
+ struct attribute **event_attr;
+ struct attribute **format_attr;
+};
+
+static struct attribute *scf_pmu_event_attrs[] = {
+ ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
+
+ ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
+ ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
+ ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
+ ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
+
+ NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
+ NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp, 0x105),
+ NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
+ NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp, 0x10d),
+ NV_CSPMU_EVENT_ATTR_4(socket, prb_data, 0x111),
+
+ NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
+ NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding, 0x119),
+ NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding, 0x11d),
+ NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding, 0x121),
+ NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding, 0x125),
+ NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding, 0x129),
+
+ NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
+ NV_CSPMU_EVENT_ATTR_4(socket, dl_access, 0x131),
+ NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
+ NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
+ NV_CSPMU_EVENT_ATTR_4(socket, ev_access, 0x13d),
+ NV_CSPMU_EVENT_ATTR_4(socket, prb_access, 0x141),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data, 0x145),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access, 0x149),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access, 0x14d),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding, 0x151),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding, 0x155),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data, 0x159),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access, 0x15d),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access, 0x161),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding, 0x165),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding, 0x169),
+
+ ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
+ ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
+ ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
+ ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp, 0x170),
+ ARM_CSPMU_EVENT_ATTR(gmem_dl_access, 0x171),
+ ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding, 0x172),
+ ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
+ ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
+ ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding, 0x175),
+ ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp, 0x176),
+ ARM_CSPMU_EVENT_ATTR(gmem_ev_access, 0x177),
+ ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding, 0x178),
+ ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
+ ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding, 0x17a),
+ ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
+
+ NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data, 0x180),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data, 0x184),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access, 0x188),
+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding, 0x18c),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data, 0x190),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data, 0x194),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access, 0x198),
+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding, 0x19c),
+
+ ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
+ ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
+
+ ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
+ ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
+ ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
+ ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp, 0x1a8),
+ ARM_CSPMU_EVENT_ATTR(cmem_dl_access, 0x1a9),
+ ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding, 0x1aa),
+ ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
+ ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
+ ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding, 0x1ad),
+ ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp, 0x1ae),
+ ARM_CSPMU_EVENT_ATTR(cmem_ev_access, 0x1af),
+ ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding, 0x1b0),
+ ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
+ ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding, 0x1b2),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data, 0x1b3),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access, 0x1b7),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access, 0x1bb),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding, 0x1bf),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding, 0x1c3),
+
+ ARM_CSPMU_EVENT_ATTR(ocu_prb_access, 0x1c7),
+ ARM_CSPMU_EVENT_ATTR(ocu_prb_data, 0x1c8),
+ ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding, 0x1c9),
+
+ ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
+
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access, 0x1cb),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data, 0x1cf),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data, 0x1d3),
+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding, 0x1d7),
+
+ ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
+
+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
+ NULL,
+};
+
+static struct attribute *mcf_pmu_event_attrs[] = {
+ ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
+ ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
+ ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
+ ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
+ ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
+ ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
+ ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
+ ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
+ ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
+ ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
+ ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
+ ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
+ ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
+ ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
+ NULL,
+};
+
+static struct attribute *generic_pmu_event_attrs[] = {
+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
+ NULL,
+};
+
+static struct attribute *scf_pmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ NULL,
+};
+
+static struct attribute *pcie_pmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
+ NULL,
+};
+
+static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ NULL,
+};
+
+static struct attribute *cnvlink_pmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
+ NULL,
+};
+
+static struct attribute *generic_pmu_format_attrs[] = {
+ ARM_CSPMU_FORMAT_EVENT_ATTR,
+ ARM_CSPMU_FORMAT_FILTER_ATTR,
+ NULL,
+};
+
+static struct attribute **
+nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
+{
+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
+
+ return ctx->event_attr;
+}
+
+static struct attribute **
+nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
+{
+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
+
+ return ctx->format_attr;
+}
+
+static const char *
+nv_cspmu_get_name(const struct arm_cspmu *cspmu)
+{
+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
+
+ return ctx->name;
+}
+
+static u32 nv_cspmu_event_filter(const struct perf_event *event)
+{
+ const struct nv_cspmu_ctx *ctx =
+ to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
+
+ if (ctx->filter_mask == 0)
+ return ctx->filter_default_val;
+
+ return event->attr.config1 & ctx->filter_mask;
+}
+
+enum nv_cspmu_name_fmt {
+ NAME_FMT_GENERIC,
+ NAME_FMT_SOCKET
+};
+
+struct nv_cspmu_match {
+ u32 prodid;
+ u32 prodid_mask;
+ u64 filter_mask;
+ u32 filter_default_val;
+ const char *name_pattern;
+ enum nv_cspmu_name_fmt name_fmt;
+ struct attribute **event_attr;
+ struct attribute **format_attr;
+};
+
+static const struct nv_cspmu_match nv_cspmu_match[] = {
+ {
+ .prodid = 0x103,
+ .prodid_mask = NV_PRODID_MASK,
+ .filter_mask = NV_PCIE_FILTER_ID_MASK,
+ .filter_default_val = NV_PCIE_FILTER_ID_MASK,
+ .name_pattern = "nvidia_pcie_pmu_%u",
+ .name_fmt = NAME_FMT_SOCKET,
+ .event_attr = mcf_pmu_event_attrs,
+ .format_attr = pcie_pmu_format_attrs
+ },
+ {
+ .prodid = 0x104,
+ .prodid_mask = NV_PRODID_MASK,
+ .filter_mask = 0x0,
+ .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
+ .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
+ .name_fmt = NAME_FMT_SOCKET,
+ .event_attr = mcf_pmu_event_attrs,
+ .format_attr = nvlink_c2c_pmu_format_attrs
+ },
+ {
+ .prodid = 0x105,
+ .prodid_mask = NV_PRODID_MASK,
+ .filter_mask = 0x0,
+ .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
+ .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
+ .name_fmt = NAME_FMT_SOCKET,
+ .event_attr = mcf_pmu_event_attrs,
+ .format_attr = nvlink_c2c_pmu_format_attrs
+ },
+ {
+ .prodid = 0x106,
+ .prodid_mask = NV_PRODID_MASK,
+ .filter_mask = NV_CNVL_FILTER_ID_MASK,
+ .filter_default_val = NV_CNVL_FILTER_ID_MASK,
+ .name_pattern = "nvidia_cnvlink_pmu_%u",
+ .name_fmt = NAME_FMT_SOCKET,
+ .event_attr = mcf_pmu_event_attrs,
+ .format_attr = cnvlink_pmu_format_attrs
+ },
+ {
+ .prodid = 0x2CF,
+ .prodid_mask = NV_PRODID_MASK,
+ .filter_mask = 0x0,
+ .filter_default_val = 0x0,
+ .name_pattern = "nvidia_scf_pmu_%u",
+ .name_fmt = NAME_FMT_SOCKET,
+ .event_attr = scf_pmu_event_attrs,
+ .format_attr = scf_pmu_format_attrs
+ },
+ {
+ .prodid = 0,
+ .prodid_mask = 0,
+ .filter_mask = NV_GENERIC_FILTER_ID_MASK,
+ .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
+ .name_pattern = "nvidia_uncore_pmu_%u",
+ .name_fmt = NAME_FMT_GENERIC,
+ .event_attr = generic_pmu_event_attrs,
+ .format_attr = generic_pmu_format_attrs
+ },
+};
+
+static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
+ const struct nv_cspmu_match *match)
+{
+ char *name;
+ struct device *dev = cspmu->dev;
+
+ static atomic_t pmu_generic_idx = {0};
+
+ switch (match->name_fmt) {
+ case NAME_FMT_SOCKET: {
+ const int cpu = cpumask_first(&cspmu->associated_cpus);
+ const int socket = cpu_to_node(cpu);
+
+ name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
+ socket);
+ break;
+ }
+ case NAME_FMT_GENERIC:
+ name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
+ atomic_fetch_inc(&pmu_generic_idx));
+ break;
+ default:
+ name = NULL;
+ break;
+ }
+
+ return name;
+}
+
+int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
+{
+ u32 prodid;
+ struct nv_cspmu_ctx *ctx;
+ struct device *dev = cspmu->dev;
+ struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
+ const struct nv_cspmu_match *match = nv_cspmu_match;
+
+ ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
+
+ /* Find matching PMU. */
+ for (; match->prodid; match++) {
+ const u32 prodid_mask = match->prodid_mask;
+
+ if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
+ break;
+ }
+
+ ctx->name = nv_cspmu_format_name(cspmu, match);
+ ctx->filter_mask = match->filter_mask;
+ ctx->filter_default_val = match->filter_default_val;
+ ctx->event_attr = match->event_attr;
+ ctx->format_attr = match->format_attr;
+
+ cspmu->impl.ctx = ctx;
+
+ /* NVIDIA specific callbacks. */
+ impl_ops->event_filter = nv_cspmu_event_filter;
+ impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
+ impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
+ impl_ops->get_name = nv_cspmu_get_name;
+
+ /* Set others to NULL to use default callback. */
+ impl_ops->event_type = NULL;
+ impl_ops->event_attr_is_visible = NULL;
+ impl_ops->get_identifier = NULL;
+ impl_ops->is_cycle_counter_event = NULL;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(nv_cspmu_init_ops);
diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.h b/drivers/perf/arm_cspmu/nvidia_cspmu.h
new file mode 100644
index 000000000000..71e18f0dc50b
--- /dev/null
+++ b/drivers/perf/arm_cspmu/nvidia_cspmu.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *
+ */
+
+/* Support for NVIDIA specific attributes. */
+
+#ifndef __NVIDIA_CSPMU_H__
+#define __NVIDIA_CSPMU_H__
+
+#include "arm_cspmu.h"
+
+/* Allocate NVIDIA descriptor. */
+int nv_cspmu_init_ops(struct arm_cspmu *cspmu);
+
+#endif /* __NVIDIA_CSPMU_H__ */
From 1830902eb896824ca313a50f3486645c2df21327 Mon Sep 17 00:00:00 2001
From: Will Deacon <will@kernel.org>
Date: Tue, 15 Nov 2022 18:24:03 +0000
Subject: [PATCH 058/151] perf: arm_cspmu: Fix modular builds due to missing
MODULE_LICENSE()s
Building an arm64 allmodconfig target results in the following failure
from modpost:
| ERROR: modpost: missing MODULE_LICENSE() in drivers/perf/arm_cspmu/arm_cspmu.o
| ERROR: modpost: missing MODULE_LICENSE() in drivers/perf/arm_cspmu/nvidia_cspmu.o
| make[1]: *** [scripts/Makefile.modpost:126: Module.symvers] Error 1
| make: *** [Makefile:1944: modpost] Error 2
Add the missing MODULE_LICENSE() macros, following the license of the
source files and symbol exports.
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_cspmu/arm_cspmu.c | 2 ++
drivers/perf/arm_cspmu/nvidia_cspmu.c | 2 ++
2 files changed, 4 insertions(+)
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c
index e851eeb33f4a..e31302ab7e37 100644
--- a/drivers/perf/arm_cspmu/arm_cspmu.c
+++ b/drivers/perf/arm_cspmu/arm_cspmu.c
@@ -1299,3 +1299,5 @@ static void __exit arm_cspmu_exit(void)
module_init(arm_cspmu_init);
module_exit(arm_cspmu_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c
index c795414ec7c7..72ef80caa3c8 100644
--- a/drivers/perf/arm_cspmu/nvidia_cspmu.c
+++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c
@@ -396,3 +396,5 @@ int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
return 0;
}
EXPORT_SYMBOL_GPL(nv_cspmu_init_ops);
+
+MODULE_LICENSE("GPL v2");
From e72dbf9085b56bbc19ff332f82adec1891077637 Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Wed, 16 Nov 2022 13:04:55 -0600
Subject: [PATCH 059/151] perf: arm_cspmu: Fix build failure on x86_64
Building on x86_64 allmodconfig failed:
| drivers/perf/arm_cspmu/arm_cspmu.c:1114:29: error: implicit
| declaration of function 'get_acpi_id_for_cpu'
get_acpi_id_for_cpu is a helper function from ARM64.
Fix by adding ARM64 dependency.
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Link: https://lore.kernel.org/r/20221116190455.55651-1-bwicaksono@nvidia.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_cspmu/Kconfig | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/perf/arm_cspmu/Kconfig b/drivers/perf/arm_cspmu/Kconfig
index 058223bef661..0b316fe69a45 100644
--- a/drivers/perf/arm_cspmu/Kconfig
+++ b/drivers/perf/arm_cspmu/Kconfig
@@ -4,7 +4,7 @@
config ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU
tristate "ARM Coresight Architecture PMU"
- depends on ACPI
+ depends on ARM64 && ACPI
depends on ACPI_APMT || COMPILE_TEST
help
Provides support for performance monitoring unit (PMU) devices
From a91bbd5c9984a2b15e68aad7a79c2809fbd10fbe Mon Sep 17 00:00:00 2001
From: Besar Wicaksono <bwicaksono@nvidia.com>
Date: Wed, 16 Nov 2022 14:39:52 -0600
Subject: [PATCH 060/151] perf: arm_cspmu: Fix module cyclic dependency
Build on arm64 allmodconfig failed with:
| depmod: ERROR: Cycle detected: arm_cspmu -> nvidia_cspmu -> arm_cspmu
| depmod: ERROR: Found 2 modules in dependency cycles!
The arm_cspmu.c provides standard functions to operate the PMU and the
vendor code provides vendor specific attributes. Both need to be built as
single kernel module.
Update the makefile to compile sources under arm_cspmu into one module.
Signed-off-by: Besar Wicaksono <bwicaksono@nvidia.com>
Reviewed-and-Tested-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Link: https://lore.kernel.org/r/20221116203952.34168-1-bwicaksono@nvidia.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_cspmu/Makefile | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/drivers/perf/arm_cspmu/Makefile b/drivers/perf/arm_cspmu/Makefile
index 641db85c018b..fedb17df982d 100644
--- a/drivers/perf/arm_cspmu/Makefile
+++ b/drivers/perf/arm_cspmu/Makefile
@@ -2,6 +2,5 @@
#
# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += \
- arm_cspmu.o \
- nvidia_cspmu.o
+obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu_module.o
+arm_cspmu_module-y := arm_cspmu.o nvidia_cspmu.o
From 9705bc70960459ae09f4a5e77283973bb3a40f57 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 3 Nov 2022 17:05:17 +0000
Subject: [PATCH 061/151] ftrace: pass fregs to arch_ftrace_set_direct_caller()
In subsequent patches we'll arrange for architectures to have an
ftrace_regs which is entirely distinct from pt_regs. In preparation for
this, we need to minimize the use of pt_regs to where strictly
necessary in the core ftrace code.
This patch changes the prototype of arch_ftrace_set_direct_caller() to
take ftrace_regs rather than pt_regs, and moves the extraction of the
pt_regs into arch_ftrace_set_direct_caller().
On x86, arch_ftrace_set_direct_caller() can be used even when
CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=n, and <linux/ftrace.h> defines
struct ftrace_regs. Due to this, it's necessary to define
arch_ftrace_set_direct_caller() as a macro to avoid using an incomplete
type. I've also moved the body of arch_ftrace_set_direct_caller() after
the CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y defineidion of struct
ftrace_regs.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Florent Revest <revest@chromium.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20221103170520.931305-2-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/s390/include/asm/ftrace.h | 5 ++++-
arch/x86/include/asm/ftrace.h | 31 ++++++++++++++++++-------------
include/linux/ftrace.h | 9 ++++-----
kernel/trace/ftrace.c | 3 +--
4 files changed, 27 insertions(+), 21 deletions(-)
diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h
index 6f80ec9c04be..85bc26ccdb87 100644
--- a/arch/s390/include/asm/ftrace.h
+++ b/arch/s390/include/asm/ftrace.h
@@ -60,6 +60,7 @@ static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *f
fregs->regs.psw.addr = ip;
}
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
/*
* When an ftrace registered caller is tracing a function that is
* also set by a register_ftrace_direct() call, it needs to be
@@ -67,10 +68,12 @@ static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *f
* place the direct caller in the ORIG_GPR2 part of pt_regs. This
* tells the ftrace_caller that there's a direct caller.
*/
-static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
+static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr)
{
+ struct pt_regs *regs = &fregs->regs;
regs->orig_gpr2 = addr;
}
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
/*
* Even though the system call numbers are identical for s390/s390x a
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 908d99b127d3..d2350a8351fa 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -34,19 +34,6 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
return addr;
}
-/*
- * When a ftrace registered caller is tracing a function that is
- * also set by a register_ftrace_direct() call, it needs to be
- * differentiated in the ftrace_caller trampoline. To do this, we
- * place the direct caller in the ORIG_AX part of pt_regs. This
- * tells the ftrace_caller that there's a direct caller.
- */
-static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
-{
- /* Emulate a call */
- regs->orig_ax = addr;
-}
-
#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
struct ftrace_regs {
struct pt_regs regs;
@@ -72,6 +59,24 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR
#endif
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+/*
+ * When a ftrace registered caller is tracing a function that is
+ * also set by a register_ftrace_direct() call, it needs to be
+ * differentiated in the ftrace_caller trampoline. To do this, we
+ * place the direct caller in the ORIG_AX part of pt_regs. This
+ * tells the ftrace_caller that there's a direct caller.
+ */
+static inline void
+__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
+{
+ /* Emulate a call */
+ regs->orig_ax = addr;
+}
+#define arch_ftrace_set_direct_caller(fregs, addr) \
+ __arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+
#ifdef CONFIG_DYNAMIC_FTRACE
struct dyn_arch_ftrace {
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 62557d4bffc2..f201fcbfffb0 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -37,9 +37,10 @@ extern void ftrace_boot_snapshot(void);
static inline void ftrace_boot_snapshot(void) { }
#endif
-#ifdef CONFIG_FUNCTION_TRACER
struct ftrace_ops;
struct ftrace_regs;
+
+#ifdef CONFIG_FUNCTION_TRACER
/*
* If the arch's mcount caller does not support all of ftrace's
* features, then it must call an indirect function that
@@ -427,9 +428,7 @@ static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsi
{
return -ENODEV;
}
-#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
-#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
/*
* This must be implemented by the architecture.
* It is the way the ftrace direct_ops helper, when called
@@ -443,9 +442,9 @@ static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsi
* the return from the trampoline jump to the direct caller
* instead of going back to the function it just traced.
*/
-static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs,
+static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
unsigned long addr) { }
-#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
#ifdef CONFIG_STACK_TRACER
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 7dc023641bf1..9eb42120ac44 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2487,14 +2487,13 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
static void call_direct_funcs(unsigned long ip, unsigned long pip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
- struct pt_regs *regs = ftrace_get_regs(fregs);
unsigned long addr;
addr = ftrace_find_rec_direct(ip);
if (!addr)
return;
- arch_ftrace_set_direct_caller(regs, addr);
+ arch_ftrace_set_direct_caller(fregs, addr);
}
struct ftrace_ops direct_ops = {
From 0ef86097f127d0d73e19aa4dcf86106105e7db09 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 3 Nov 2022 17:05:18 +0000
Subject: [PATCH 062/151] ftrace: rename ftrace_instruction_pointer_set() ->
ftrace_regs_set_instruction_pointer()
In subsequent patches we'll add a sew of ftrace_regs_{get,set}_*()
helpers. In preparation, this patch renames
ftrace_instruction_pointer_set() to
ftrace_regs_set_instruction_pointer().
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Florent Revest <revest@chromium.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20221103170520.931305-3-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/powerpc/include/asm/ftrace.h | 5 +++--
arch/s390/include/asm/ftrace.h | 5 +++--
arch/x86/include/asm/ftrace.h | 2 +-
include/linux/ftrace.h | 9 ++++-----
kernel/livepatch/patch.c | 2 +-
5 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index 3cee7115441b..c3eb48f67566 100644
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -37,8 +37,9 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
return fregs->regs.msr ? &fregs->regs : NULL;
}
-static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *fregs,
- unsigned long ip)
+static __always_inline void
+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
+ unsigned long ip)
{
regs_set_return_ip(&fregs->regs, ip);
}
diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h
index 85bc26ccdb87..113d194acea0 100644
--- a/arch/s390/include/asm/ftrace.h
+++ b/arch/s390/include/asm/ftrace.h
@@ -54,8 +54,9 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
return NULL;
}
-static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *fregs,
- unsigned long ip)
+static __always_inline void
+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
+ unsigned long ip)
{
fregs->regs.psw.addr = ip;
}
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index d2350a8351fa..b2925d342c65 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -48,7 +48,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs)
return &fregs->regs;
}
-#define ftrace_instruction_pointer_set(fregs, _ip) \
+#define ftrace_regs_set_instruction_pointer(fregs, _ip) \
do { (fregs)->regs.ip = (_ip); } while (0)
struct ftrace_ops;
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index f201fcbfffb0..ec3657a60f85 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -111,12 +111,11 @@ struct ftrace_regs {
#define arch_ftrace_get_regs(fregs) (&(fregs)->regs)
/*
- * ftrace_instruction_pointer_set() is to be defined by the architecture
- * if to allow setting of the instruction pointer from the ftrace_regs
- * when HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports
- * live kernel patching.
+ * ftrace_regs_set_instruction_pointer() is to be defined by the architecture
+ * if to allow setting of the instruction pointer from the ftrace_regs when
+ * HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports live kernel patching.
*/
-#define ftrace_instruction_pointer_set(fregs, ip) do { } while (0)
+#define ftrace_regs_set_instruction_pointer(fregs, ip) do { } while (0)
#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
diff --git a/kernel/livepatch/patch.c b/kernel/livepatch/patch.c
index 4c4f5a776d80..4152c71507e2 100644
--- a/kernel/livepatch/patch.c
+++ b/kernel/livepatch/patch.c
@@ -118,7 +118,7 @@ static void notrace klp_ftrace_handler(unsigned long ip,
if (func->nop)
goto unlock;
- ftrace_instruction_pointer_set(fregs, (unsigned long)func->new_func);
+ ftrace_regs_set_instruction_pointer(fregs, (unsigned long)func->new_func);
unlock:
ftrace_test_recursion_unlock(bit);
From 94d095ffa0e16bb7f161a2b73bbe5c2795d499a8 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 3 Nov 2022 17:05:19 +0000
Subject: [PATCH 063/151] ftrace: abstract DYNAMIC_FTRACE_WITH_ARGS accesses
In subsequent patches we'll arrange for architectures to have an
ftrace_regs which is entirely distinct from pt_regs. In preparation for
this, we need to minimize the use of pt_regs to where strictly necessary
in the core ftrace code.
This patch adds new ftrace_regs_{get,set}_*() helpers which can be used
to manipulate ftrace_regs. When CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y,
these can always be used on any ftrace_regs, and when
CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=n these can be used when regs are
available. A new ftrace_regs_has_args(fregs) helper is added which code
can use to check when these are usable.
Co-developed-by: Florent Revest <revest@chromium.org>
Signed-off-by: Florent Revest <revest@chromium.org>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20221103170520.931305-4-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/powerpc/include/asm/ftrace.h | 19 +++++++++++++++++++
arch/s390/include/asm/ftrace.h | 19 +++++++++++++++++++
arch/x86/include/asm/ftrace.h | 16 ++++++++++++++++
include/linux/ftrace.h | 29 +++++++++++++++++++++++++++++
kernel/trace/Kconfig | 6 +++---
5 files changed, 86 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index c3eb48f67566..259b9dd5fe1c 100644
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -44,6 +44,25 @@ ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
regs_set_return_ip(&fregs->regs, ip);
}
+static __always_inline unsigned long
+ftrace_regs_get_instruction_pointer(struct ftrace_regs *fregs)
+{
+ return instruction_pointer(&fregs->regs);
+}
+
+#define ftrace_regs_get_argument(fregs, n) \
+ regs_get_kernel_argument(&(fregs)->regs, n)
+#define ftrace_regs_get_stack_pointer(fregs) \
+ kernel_stack_pointer(&(fregs)->regs)
+#define ftrace_regs_return_value(fregs) \
+ regs_return_value(&(fregs)->regs)
+#define ftrace_regs_set_return_value(fregs, ret) \
+ regs_set_return_value(&(fregs)->regs, ret)
+#define ftrace_override_function_with_return(fregs) \
+ override_function_with_return(&(fregs)->regs)
+#define ftrace_regs_query_register_offset(name) \
+ regs_query_register_offset(name)
+
struct ftrace_ops;
#define ftrace_graph_func ftrace_graph_func
diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h
index 113d194acea0..e5c5cb1207e2 100644
--- a/arch/s390/include/asm/ftrace.h
+++ b/arch/s390/include/asm/ftrace.h
@@ -54,6 +54,12 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *
return NULL;
}
+static __always_inline unsigned long
+ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs)
+{
+ return fregs->regs.psw.addr;
+}
+
static __always_inline void
ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
unsigned long ip)
@@ -61,6 +67,19 @@ ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
fregs->regs.psw.addr = ip;
}
+#define ftrace_regs_get_argument(fregs, n) \
+ regs_get_kernel_argument(&(fregs)->regs, n)
+#define ftrace_regs_get_stack_pointer(fregs) \
+ kernel_stack_pointer(&(fregs)->regs)
+#define ftrace_regs_return_value(fregs) \
+ regs_return_value(&(fregs)->regs)
+#define ftrace_regs_set_return_value(fregs, ret) \
+ regs_set_return_value(&(fregs)->regs, ret)
+#define ftrace_override_function_with_return(fregs) \
+ override_function_with_return(&(fregs)->regs)
+#define ftrace_regs_query_register_offset(name) \
+ regs_query_register_offset(name)
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
/*
* When an ftrace registered caller is tracing a function that is
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b2925d342c65..5061ac98ffa1 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -51,6 +51,22 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs)
#define ftrace_regs_set_instruction_pointer(fregs, _ip) \
do { (fregs)->regs.ip = (_ip); } while (0)
+#define ftrace_regs_get_instruction_pointer(fregs) \
+ ((fregs)->regs.ip)
+
+#define ftrace_regs_get_argument(fregs, n) \
+ regs_get_kernel_argument(&(fregs)->regs, n)
+#define ftrace_regs_get_stack_pointer(fregs) \
+ kernel_stack_pointer(&(fregs)->regs)
+#define ftrace_regs_return_value(fregs) \
+ regs_return_value(&(fregs)->regs)
+#define ftrace_regs_set_return_value(fregs, ret) \
+ regs_set_return_value(&(fregs)->regs, ret)
+#define ftrace_override_function_with_return(fregs) \
+ override_function_with_return(&(fregs)->regs)
+#define ftrace_regs_query_register_offset(name) \
+ regs_query_register_offset(name)
+
struct ftrace_ops;
#define ftrace_graph_func ftrace_graph_func
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index ec3657a60f85..99f1146614c0 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -126,6 +126,35 @@ static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs
return arch_ftrace_get_regs(fregs);
}
+/*
+ * When true, the ftrace_regs_{get,set}_*() functions may be used on fregs.
+ * Note: this can be true even when ftrace_get_regs() cannot provide a pt_regs.
+ */
+static __always_inline bool ftrace_regs_has_args(struct ftrace_regs *fregs)
+{
+ if (IS_ENABLED(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS))
+ return true;
+
+ return ftrace_get_regs(fregs) != NULL;
+}
+
+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
+#define ftrace_regs_get_instruction_pointer(fregs) \
+ instruction_pointer(ftrace_get_regs(fregs))
+#define ftrace_regs_get_argument(fregs, n) \
+ regs_get_kernel_argument(ftrace_get_regs(fregs), n)
+#define ftrace_regs_get_stack_pointer(fregs) \
+ kernel_stack_pointer(ftrace_get_regs(fregs))
+#define ftrace_regs_return_value(fregs) \
+ regs_return_value(ftrace_get_regs(fregs))
+#define ftrace_regs_set_return_value(fregs, ret) \
+ regs_set_return_value(ftrace_get_regs(fregs), ret)
+#define ftrace_override_function_with_return(fregs) \
+ override_function_with_return(ftrace_get_regs(fregs))
+#define ftrace_regs_query_register_offset(name) \
+ regs_query_register_offset(name)
+#endif
+
typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index e9e95c790b8e..2c6611c13f99 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -46,10 +46,10 @@ config HAVE_DYNAMIC_FTRACE_WITH_ARGS
bool
help
If this is set, then arguments and stack can be found from
- the pt_regs passed into the function callback regs parameter
+ the ftrace_regs passed into the function callback regs parameter
by default, even without setting the REGS flag in the ftrace_ops.
- This allows for use of regs_get_kernel_argument() and
- kernel_stack_pointer().
+ This allows for use of ftrace_regs_get_argument() and
+ ftrace_regs_get_stack_pointer().
config HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
bool
From 26299b3f6ba26bfc234b73126d14bdf4dec5275a Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 3 Nov 2022 17:05:20 +0000
Subject: [PATCH 064/151] ftrace: arm64: move from REGS to ARGS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This commit replaces arm64's support for FTRACE_WITH_REGS with support
for FTRACE_WITH_ARGS. This removes some overhead and complexity, and
removes some latent issues with inconsistent presentation of struct
pt_regs (which can only be reliably saved/restored at exception
boundaries).
FTRACE_WITH_REGS has been supported on arm64 since commit:
3b23e4991fb66f6d ("arm64: implement ftrace with regs")
As noted in the commit message, the major reasons for implementing
FTRACE_WITH_REGS were:
(1) To make it possible to use the ftrace graph tracer with pointer
authentication, where it's necessary to snapshot/manipulate the LR
before it is signed by the instrumented function.
(2) To make it possible to implement LIVEPATCH in future, where we need
to hook function entry before an instrumented function manipulates
the stack or argument registers. Practically speaking, we need to
preserve the argument/return registers, PC, LR, and SP.
Neither of these need a struct pt_regs, and only require the set of
registers which are live at function call/return boundaries. Our calling
convention is defined by "Procedure Call Standard for the Arm® 64-bit
Architecture (AArch64)" (AKA "AAPCS64"), which can currently be found
at:
https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst
Per AAPCS64, all function call argument and return values are held in
the following GPRs:
* X0 - X7 : parameter / result registers
* X8 : indirect result location register
* SP : stack pointer (AKA SP)
Additionally, ad function call boundaries, the following GPRs hold
context/return information:
* X29 : frame pointer (AKA FP)
* X30 : link register (AKA LR)
... and for ftrace we need to capture the instrumented address:
* PC : program counter
No other GPRs are relevant, as none of the other arguments hold
parameters or return values:
* X9 - X17 : temporaries, may be clobbered
* X18 : shadow call stack pointer (or temorary)
* X19 - X28 : callee saved
This patch implements FTRACE_WITH_ARGS for arm64, only saving/restoring
the minimal set of registers necessary. This is always sufficient to
manipulate control flow (e.g. for live-patching) or to manipulate
function arguments and return values.
This reduces the necessary stack usage from 336 bytes for pt_regs down
to 112 bytes for ftrace_regs + 32 bytes for two frame records, freeing
up 188 bytes. This could be reduced further with changes to the
unwinder.
As there is no longer a need to save different sets of registers for
different features, we no longer need distinct `ftrace_caller` and
`ftrace_regs_caller` trampolines. This allows the trampoline assembly to
be simpler, and simplifies code which previously had to handle the two
trampolines.
I've tested this with the ftrace selftests, where there are no
unexpected failures.
Co-developed-by: Florent Revest <revest@chromium.org>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Florent Revest <revest@chromium.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20221103170520.931305-5-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 18 +++--
arch/arm64/Makefile | 2 +-
arch/arm64/include/asm/ftrace.h | 72 ++++++++++++++++--
arch/arm64/kernel/asm-offsets.c | 13 ++++
arch/arm64/kernel/entry-ftrace.S | 123 ++++++++++++-------------------
arch/arm64/kernel/ftrace.c | 82 ++++++++++++---------
arch/arm64/kernel/module.c | 3 -
7 files changed, 187 insertions(+), 126 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 505c8a1ccbe0..b6b3305ba701 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -181,8 +181,10 @@ config ARM64
select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_CONTIGUOUS
select HAVE_DYNAMIC_FTRACE
+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS \
+ if $(cc-option,-fpatchable-function-entry=2)
select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \
- if DYNAMIC_FTRACE_WITH_REGS
+ if DYNAMIC_FTRACE_WITH_ARGS
select HAVE_EFFICIENT_UNALIGNED_ACCESS
select HAVE_FAST_GUP
select HAVE_FTRACE_MCOUNT_RECORD
@@ -233,16 +235,16 @@ config ARM64
help
ARM 64-bit (AArch64) Linux support.
-config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_REGS
+config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
def_bool CC_IS_CLANG
# https://github.com/ClangBuiltLinux/linux/issues/1507
depends on AS_IS_GNU || (AS_IS_LLVM && (LD_IS_LLD || LD_VERSION >= 23600))
- select HAVE_DYNAMIC_FTRACE_WITH_REGS
+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS
-config GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_REGS
+config GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
def_bool CC_IS_GCC
depends on $(cc-option,-fpatchable-function-entry=2)
- select HAVE_DYNAMIC_FTRACE_WITH_REGS
+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS
config 64BIT
def_bool y
@@ -1816,7 +1818,7 @@ config ARM64_PTR_AUTH_KERNEL
# which is only understood by binutils starting with version 2.33.1.
depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100)
depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE
- depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
+ depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS)
help
If the compiler supports the -mbranch-protection or
-msign-return-address flag (e.g. GCC 7 or later), then this option
@@ -1826,7 +1828,7 @@ config ARM64_PTR_AUTH_KERNEL
disabled with minimal loss of protection.
This feature works with FUNCTION_GRAPH_TRACER option only if
- DYNAMIC_FTRACE_WITH_REGS is enabled.
+ DYNAMIC_FTRACE_WITH_ARGS is enabled.
config CC_HAS_BRANCH_PROT_PAC_RET
# GCC 9 or later, clang 8 or later
@@ -1924,7 +1926,7 @@ config ARM64_BTI_KERNEL
depends on !CC_IS_GCC
# https://github.com/llvm/llvm-project/commit/a88c722e687e6780dcd6a58718350dc76fcc4cc9
depends on !CC_IS_CLANG || CLANG_VERSION >= 120000
- depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
+ depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS)
help
Build the kernel with Branch Target Identification annotations
and enable enforcement of this for kernel code. When this option
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 5e56d26a2239..b1202fa84bab 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -128,7 +128,7 @@ endif
CHECKFLAGS += -D__aarch64__
-ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_REGS),y)
+ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_ARGS),y)
KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
CC_FLAGS_FTRACE := -fpatchable-function-entry=2
endif
diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h
index 329dbbd4d50b..5664729800ae 100644
--- a/arch/arm64/include/asm/ftrace.h
+++ b/arch/arm64/include/asm/ftrace.h
@@ -23,7 +23,7 @@
*/
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
#define ARCH_SUPPORTS_FTRACE_OPS 1
#else
#define MCOUNT_ADDR ((unsigned long)_mcount)
@@ -33,8 +33,7 @@
#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE
#define FTRACE_PLT_IDX 0
-#define FTRACE_REGS_PLT_IDX 1
-#define NR_FTRACE_PLTS 2
+#define NR_FTRACE_PLTS 1
/*
* Currently, gcc tends to save the link register after the local variables
@@ -69,7 +68,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
* Adjust addr to point at the BL in the callsite.
* See ftrace_init_nop() for the callsite sequence.
*/
- if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
+ if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS))
return addr + AARCH64_INSN_SIZE;
/*
* addr is the address of the mcount call instruction.
@@ -78,10 +77,71 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
return addr;
}
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
struct dyn_ftrace;
struct ftrace_ops;
-struct ftrace_regs;
+
+#define arch_ftrace_get_regs(regs) NULL
+
+struct ftrace_regs {
+ /* x0 - x8 */
+ unsigned long regs[9];
+ unsigned long __unused;
+
+ unsigned long fp;
+ unsigned long lr;
+
+ unsigned long sp;
+ unsigned long pc;
+};
+
+static __always_inline unsigned long
+ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs)
+{
+ return fregs->pc;
+}
+
+static __always_inline void
+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
+ unsigned long pc)
+{
+ fregs->pc = pc;
+}
+
+static __always_inline unsigned long
+ftrace_regs_get_stack_pointer(const struct ftrace_regs *fregs)
+{
+ return fregs->sp;
+}
+
+static __always_inline unsigned long
+ftrace_regs_get_argument(struct ftrace_regs *fregs, unsigned int n)
+{
+ if (n < 8)
+ return fregs->regs[n];
+ return 0;
+}
+
+static __always_inline unsigned long
+ftrace_regs_get_return_value(const struct ftrace_regs *fregs)
+{
+ return fregs->regs[0];
+}
+
+static __always_inline void
+ftrace_regs_set_return_value(struct ftrace_regs *fregs,
+ unsigned long ret)
+{
+ fregs->regs[0] = ret;
+}
+
+static __always_inline void
+ftrace_override_function_with_return(struct ftrace_regs *fregs)
+{
+ fregs->pc = fregs->lr;
+}
+
+int ftrace_regs_query_register_offset(const char *name);
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
#define ftrace_init_nop ftrace_init_nop
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 1197e7679882..2234624536d9 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -82,6 +82,19 @@ int main(void)
DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs));
BLANK();
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
+ DEFINE(FREGS_X0, offsetof(struct ftrace_regs, regs[0]));
+ DEFINE(FREGS_X2, offsetof(struct ftrace_regs, regs[2]));
+ DEFINE(FREGS_X4, offsetof(struct ftrace_regs, regs[4]));
+ DEFINE(FREGS_X6, offsetof(struct ftrace_regs, regs[6]));
+ DEFINE(FREGS_X8, offsetof(struct ftrace_regs, regs[8]));
+ DEFINE(FREGS_FP, offsetof(struct ftrace_regs, fp));
+ DEFINE(FREGS_LR, offsetof(struct ftrace_regs, lr));
+ DEFINE(FREGS_SP, offsetof(struct ftrace_regs, sp));
+ DEFINE(FREGS_PC, offsetof(struct ftrace_regs, pc));
+ DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs));
+ BLANK();
+#endif
#ifdef CONFIG_COMPAT
DEFINE(COMPAT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_sigframe, uc.uc_mcontext.arm_r0));
DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_rt_sigframe, sig.uc.uc_mcontext.arm_r0));
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 795344ab4ec4..4d3050549aa6 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -13,83 +13,58 @@
#include <asm/ftrace.h>
#include <asm/insn.h>
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
/*
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
* ftrace_make_call() have patched those NOPs to:
*
* MOV X9, LR
- * BL <entry>
- *
- * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
+ * BL ftrace_caller
*
* Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
* live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
* clobber.
*
- * We save the callsite's context into a pt_regs before invoking any ftrace
- * callbacks. So that we can get a sensible backtrace, we create a stack record
- * for the callsite and the ftrace entry assembly. This is not sufficient for
- * reliable stacktrace: until we create the callsite stack record, its caller
- * is missing from the LR and existing chain of frame records.
+ * We save the callsite's context into a struct ftrace_regs before invoking any
+ * ftrace callbacks. So that we can get a sensible backtrace, we create frame
+ * records for the callsite and the ftrace entry assembly. This is not
+ * sufficient for reliable stacktrace: until we create the callsite stack
+ * record, its caller is missing from the LR and existing chain of frame
+ * records.
*/
- .macro ftrace_regs_entry, allregs=0
- /* Make room for pt_regs, plus a callee frame */
- sub sp, sp, #(PT_REGS_SIZE + 16)
-
- /* Save function arguments (and x9 for simplicity) */
- stp x0, x1, [sp, #S_X0]
- stp x2, x3, [sp, #S_X2]
- stp x4, x5, [sp, #S_X4]
- stp x6, x7, [sp, #S_X6]
- stp x8, x9, [sp, #S_X8]
-
- /* Optionally save the callee-saved registers, always save the FP */
- .if \allregs == 1
- stp x10, x11, [sp, #S_X10]
- stp x12, x13, [sp, #S_X12]
- stp x14, x15, [sp, #S_X14]
- stp x16, x17, [sp, #S_X16]
- stp x18, x19, [sp, #S_X18]
- stp x20, x21, [sp, #S_X20]
- stp x22, x23, [sp, #S_X22]
- stp x24, x25, [sp, #S_X24]
- stp x26, x27, [sp, #S_X26]
- stp x28, x29, [sp, #S_X28]
- .else
- str x29, [sp, #S_FP]
- .endif
-
- /* Save the callsite's SP and LR */
- add x10, sp, #(PT_REGS_SIZE + 16)
- stp x9, x10, [sp, #S_LR]
-
- /* Save the PC after the ftrace callsite */
- str x30, [sp, #S_PC]
-
- /* Create a frame record for the callsite above pt_regs */
- stp x29, x9, [sp, #PT_REGS_SIZE]
- add x29, sp, #PT_REGS_SIZE
-
- /* Create our frame record within pt_regs. */
- stp x29, x30, [sp, #S_STACKFRAME]
- add x29, sp, #S_STACKFRAME
- .endm
-
-SYM_CODE_START(ftrace_regs_caller)
- bti c
- ftrace_regs_entry 1
- b ftrace_common
-SYM_CODE_END(ftrace_regs_caller)
-
SYM_CODE_START(ftrace_caller)
bti c
- ftrace_regs_entry 0
- b ftrace_common
-SYM_CODE_END(ftrace_caller)
-SYM_CODE_START(ftrace_common)
+ /* Save original SP */
+ mov x10, sp
+
+ /* Make room for ftrace regs, plus two frame records */
+ sub sp, sp, #(FREGS_SIZE + 32)
+
+ /* Save function arguments */
+ stp x0, x1, [sp, #FREGS_X0]
+ stp x2, x3, [sp, #FREGS_X2]
+ stp x4, x5, [sp, #FREGS_X4]
+ stp x6, x7, [sp, #FREGS_X6]
+ str x8, [sp, #FREGS_X8]
+
+ /* Save the callsite's FP, LR, SP */
+ str x29, [sp, #FREGS_FP]
+ str x9, [sp, #FREGS_LR]
+ str x10, [sp, #FREGS_SP]
+
+ /* Save the PC after the ftrace callsite */
+ str x30, [sp, #FREGS_PC]
+
+ /* Create a frame record for the callsite above the ftrace regs */
+ stp x29, x9, [sp, #FREGS_SIZE + 16]
+ add x29, sp, #FREGS_SIZE + 16
+
+ /* Create our frame record above the ftrace regs */
+ stp x29, x30, [sp, #FREGS_SIZE]
+ add x29, sp, #FREGS_SIZE
+
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
mov x1, x9 // parent_ip (callsite's LR)
ldr_l x2, function_trace_op // op
@@ -104,24 +79,24 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
* to restore x0-x8, x29, and x30.
*/
/* Restore function arguments */
- ldp x0, x1, [sp]
- ldp x2, x3, [sp, #S_X2]
- ldp x4, x5, [sp, #S_X4]
- ldp x6, x7, [sp, #S_X6]
- ldr x8, [sp, #S_X8]
+ ldp x0, x1, [sp, #FREGS_X0]
+ ldp x2, x3, [sp, #FREGS_X2]
+ ldp x4, x5, [sp, #FREGS_X4]
+ ldp x6, x7, [sp, #FREGS_X6]
+ ldr x8, [sp, #FREGS_X8]
/* Restore the callsite's FP, LR, PC */
- ldr x29, [sp, #S_FP]
- ldr x30, [sp, #S_LR]
- ldr x9, [sp, #S_PC]
+ ldr x29, [sp, #FREGS_FP]
+ ldr x30, [sp, #FREGS_LR]
+ ldr x9, [sp, #FREGS_PC]
/* Restore the callsite's SP */
- add sp, sp, #PT_REGS_SIZE + 16
+ add sp, sp, #FREGS_SIZE + 32
ret x9
-SYM_CODE_END(ftrace_common)
+SYM_CODE_END(ftrace_caller)
-#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
+#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
/*
* Gcc with -pg will put the following code in the beginning of each function:
@@ -293,7 +268,7 @@ SYM_FUNC_START(ftrace_graph_caller)
mcount_exit
SYM_FUNC_END(ftrace_graph_caller)
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
-#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
SYM_TYPED_FUNC_START(ftrace_stub)
ret
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index 8745175f4a75..5cf990d052ba 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -17,6 +17,49 @@
#include <asm/insn.h>
#include <asm/patching.h>
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
+struct fregs_offset {
+ const char *name;
+ int offset;
+};
+
+#define FREGS_OFFSET(n, field) \
+{ \
+ .name = n, \
+ .offset = offsetof(struct ftrace_regs, field), \
+}
+
+static const struct fregs_offset fregs_offsets[] = {
+ FREGS_OFFSET("x0", regs[0]),
+ FREGS_OFFSET("x1", regs[1]),
+ FREGS_OFFSET("x2", regs[2]),
+ FREGS_OFFSET("x3", regs[3]),
+ FREGS_OFFSET("x4", regs[4]),
+ FREGS_OFFSET("x5", regs[5]),
+ FREGS_OFFSET("x6", regs[6]),
+ FREGS_OFFSET("x7", regs[7]),
+ FREGS_OFFSET("x8", regs[8]),
+
+ FREGS_OFFSET("x29", fp),
+ FREGS_OFFSET("x30", lr),
+ FREGS_OFFSET("lr", lr),
+
+ FREGS_OFFSET("sp", sp),
+ FREGS_OFFSET("pc", pc),
+};
+
+int ftrace_regs_query_register_offset(const char *name)
+{
+ for (int i = 0; i < ARRAY_SIZE(fregs_offsets); i++) {
+ const struct fregs_offset *roff = &fregs_offsets[i];
+ if (!strcmp(roff->name, name))
+ return roff->offset;
+ }
+
+ return -EINVAL;
+}
+#endif
+
#ifdef CONFIG_DYNAMIC_FTRACE
/*
* Replace a single instruction, which may be a branch or NOP.
@@ -70,9 +113,6 @@ static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
if (addr == FTRACE_ADDR)
return &plt[FTRACE_PLT_IDX];
- if (addr == FTRACE_REGS_ADDR &&
- IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
- return &plt[FTRACE_REGS_PLT_IDX];
#endif
return NULL;
}
@@ -154,25 +194,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
return ftrace_modify_code(pc, old, new, true);
}
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
-int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
- unsigned long addr)
-{
- unsigned long pc = rec->ip;
- u32 old, new;
-
- if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
- return -EINVAL;
- if (!ftrace_find_callable_addr(rec, NULL, &addr))
- return -EINVAL;
-
- old = aarch64_insn_gen_branch_imm(pc, old_addr,
- AARCH64_INSN_BRANCH_LINK);
- new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
-
- return ftrace_modify_code(pc, old, new, true);
-}
-
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
/*
* The compiler has inserted two NOPs before the regular function prologue.
* All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
@@ -228,7 +250,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
*
* Note: 'mod' is only set at module load time.
*/
- if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) &&
+ if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) &&
IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) {
return aarch64_insn_patch_text_nosync((void *)pc, new);
}
@@ -279,19 +301,11 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
#ifdef CONFIG_DYNAMIC_FTRACE
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
- /*
- * When DYNAMIC_FTRACE_WITH_REGS is selected, `fregs` can never be NULL
- * and arch_ftrace_get_regs(fregs) will always give a non-NULL pt_regs
- * in which we can safely modify the LR.
- */
- struct pt_regs *regs = arch_ftrace_get_regs(fregs);
- unsigned long *parent = (unsigned long *)&procedure_link_pointer(regs);
-
- prepare_ftrace_return(ip, parent, frame_pointer(regs));
+ prepare_ftrace_return(ip, &fregs->lr, fregs->fp);
}
#else
/*
@@ -323,6 +337,6 @@ int ftrace_disable_ftrace_graph_caller(void)
{
return ftrace_modify_graph_caller(false);
}
-#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 76b41e4ca9fa..acd0d883e9ca 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -497,9 +497,6 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr,
__init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR);
- if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
- __init_plt(&plts[FTRACE_REGS_PLT_IDX], FTRACE_REGS_ADDR);
-
mod->arch.ftrace_trampolines = plts;
#endif
return 0;
From 9ed2b4616d4e846ece2a04cb5007ce1d1bd9e3f3 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Fri, 18 Nov 2022 11:01:02 +0530
Subject: [PATCH 065/151] arm64/mm: Drop redundant BUG_ON(!pgtable_alloc)
__create_pgd_mapping_locked() expects a page allocator used while mapping a
virtual range. This page allocator function propagates down the call chain,
while building intermediate levels in the page table. Passed page allocator
is a necessary ingredient required to build the page table but its presence
can be asserted just once in the very beginning rather than in all the down
stream functions. This consolidates BUG_ON(!pgtable_alloc) checks just in a
single place i.e __create_pgd_mapping_locked().
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221118053102.500216-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/mm/mmu.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d386033a074c..73a12b3abf82 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -207,7 +207,6 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
if (flags & NO_EXEC_MAPPINGS)
pmdval |= PMD_TABLE_PXN;
- BUG_ON(!pgtable_alloc);
pte_phys = pgtable_alloc(PAGE_SHIFT);
__pmd_populate(pmdp, pte_phys, pmdval);
pmd = READ_ONCE(*pmdp);
@@ -285,7 +284,6 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
if (flags & NO_EXEC_MAPPINGS)
pudval |= PUD_TABLE_PXN;
- BUG_ON(!pgtable_alloc);
pmd_phys = pgtable_alloc(PMD_SHIFT);
__pud_populate(pudp, pmd_phys, pudval);
pud = READ_ONCE(*pudp);
@@ -324,7 +322,6 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
if (flags & NO_EXEC_MAPPINGS)
p4dval |= P4D_TABLE_PXN;
- BUG_ON(!pgtable_alloc);
pud_phys = pgtable_alloc(PUD_SHIFT);
__p4d_populate(p4dp, pud_phys, p4dval);
p4d = READ_ONCE(*p4dp);
@@ -383,6 +380,7 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
phys &= PAGE_MASK;
addr = virt & PAGE_MASK;
end = PAGE_ALIGN(virt + size);
+ BUG_ON(!pgtable_alloc);
do {
next = pgd_addr_end(addr, end);
From d3d10f0d370c6c83e88cd79e450e70c31c6ae50c Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Wed, 16 Nov 2022 14:13:02 +0530
Subject: [PATCH 066/151] arm64/mm: Drop idmap_pg_end[] declaration
idmap_pg_end[] is not used anywhere, hence just drop its declaration.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lore.kernel.org/r/20221116084302.320685-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/pgtable.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 71a1af42f0e8..3efedacc014f 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -609,7 +609,6 @@ extern pgd_t init_pg_dir[PTRS_PER_PGD];
extern pgd_t init_pg_end[];
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
-extern pgd_t idmap_pg_end[];
extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
extern pgd_t reserved_pg_dir[PTRS_PER_PGD];
From a149cf00b158e1793a8dd89ca492379c366300d2 Mon Sep 17 00:00:00 2001
From: Zhen Lei <thunder.leizhen@huawei.com>
Date: Wed, 16 Nov 2022 20:10:43 +0800
Subject: [PATCH 067/151] arm64: kdump: Provide default size when
crashkernel=Y,low is not specified
Try to allocate at least 128 MiB low memory automatically for the case
that crashkernel=,high is explicitly specified, while crashkenrel=,low
is omitted. This allows users to focus more on the high memory
requirements of their business rather than the low memory requirements
of the crash kernel booting.
Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
Acked-by: Baoquan He <bhe@redhat.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221116121044.1690-2-thunder.leizhen@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/admin-guide/kernel-parameters.txt | 13 +++++--------
arch/arm64/mm/init.c | 8 ++++++--
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..6c6fafc3fba9 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -851,26 +851,23 @@
available.
It will be ignored if crashkernel=X is specified.
crashkernel=size[KMG],low
- [KNL, X86-64] range under 4G. When crashkernel=X,high
+ [KNL, X86-64, ARM64] range under 4G. When crashkernel=X,high
is passed, kernel could allocate physical memory region
above 4G, that cause second kernel crash on system
that require some amount of low memory, e.g. swiotlb
requires at least 64M+32K low memory, also enough extra
low memory is needed to make sure DMA buffers for 32-bit
devices won't run out. Kernel would try to allocate
- at least 256M below 4G automatically.
+ default size of memory below 4G automatically. The default
+ size is platform dependent.
+ --> x86: max(swiotlb_size_or_default() + 8MiB, 256MiB)
+ --> arm64: 128MiB
This one lets the user specify own low range under 4G
for second kernel instead.
0: to disable low allocation.
It will be ignored when crashkernel=X,high is not used
or memory reserved is below 4G.
- [KNL, ARM64] range in low memory.
- This one lets the user specify a low range in the
- DMA zone for the crash dump kernel.
- It will be ignored when crashkernel=X,high is not used
- or memory reserved is located in the DMA zones.
-
cryptomgr.notests
[KNL] Disable crypto self-tests
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 4b4651ee47f2..ba7227179822 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -96,6 +96,8 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit = PHYS_MASK + 1;
#define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit
#define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1)
+#define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20)
+
static int __init reserve_crashkernel_low(unsigned long long low_size)
{
unsigned long long low_base;
@@ -147,7 +149,9 @@ static void __init reserve_crashkernel(void)
* is not allowed.
*/
ret = parse_crashkernel_low(cmdline, 0, &crash_low_size, &crash_base);
- if (ret && (ret != -ENOENT))
+ if (ret == -ENOENT)
+ crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE;
+ else if (ret)
return;
crash_max = CRASH_ADDR_HIGH_MAX;
@@ -170,7 +174,7 @@ static void __init reserve_crashkernel(void)
return;
}
- if ((crash_base >= CRASH_ADDR_LOW_MAX) &&
+ if ((crash_base > CRASH_ADDR_LOW_MAX - crash_low_size) &&
crash_low_size && reserve_crashkernel_low(crash_low_size)) {
memblock_phys_free(crash_base, crash_size);
return;
From a9ae89df737756d92f0e14873339cf393f7f7eb0 Mon Sep 17 00:00:00 2001
From: Zhen Lei <thunder.leizhen@huawei.com>
Date: Wed, 16 Nov 2022 20:10:44 +0800
Subject: [PATCH 068/151] arm64: kdump: Support crashkernel=X fall back to
reserve region above DMA zones
For crashkernel=X without '@offset', select a region within DMA zones
first, and fall back to reserve region above DMA zones. This allows
users to use the same configuration on multiple platforms.
Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
Acked-by: Baoquan He <bhe@redhat.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221116121044.1690-3-thunder.leizhen@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/admin-guide/kernel-parameters.txt | 2 +-
arch/arm64/mm/init.c | 17 ++++++++++++++++-
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 6c6fafc3fba9..9d458dfbec5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -831,7 +831,7 @@
memory region [offset, offset + size] for that kernel
image. If '@offset' is omitted, then a suitable offset
is selected automatically.
- [KNL, X86-64] Select a region under 4G first, and
+ [KNL, X86-64, ARM64] Select a region under 4G first, and
fall back to reserve region above 4G when '@offset'
hasn't been specified.
See Documentation/admin-guide/kdump/kdump.rst for further details.
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index ba7227179822..58a0bb2c17f1 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -132,6 +132,7 @@ static void __init reserve_crashkernel(void)
unsigned long long crash_max = CRASH_ADDR_LOW_MAX;
char *cmdline = boot_command_line;
int ret;
+ bool fixed_base = false;
if (!IS_ENABLED(CONFIG_KEXEC_CORE))
return;
@@ -163,12 +164,26 @@ static void __init reserve_crashkernel(void)
crash_size = PAGE_ALIGN(crash_size);
/* User specifies base address explicitly. */
- if (crash_base)
+ if (crash_base) {
+ fixed_base = true;
crash_max = crash_base + crash_size;
+ }
+retry:
crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN,
crash_base, crash_max);
if (!crash_base) {
+ /*
+ * If the first attempt was for low memory, fall back to
+ * high memory, the minimum required low memory will be
+ * reserved later.
+ */
+ if (!fixed_base && (crash_max == CRASH_ADDR_LOW_MAX)) {
+ crash_max = CRASH_ADDR_HIGH_MAX;
+ crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE;
+ goto retry;
+ }
+
pr_warn("cannot allocate crashkernel (size:0x%llx)\n",
crash_size);
return;
From 56eea7f87fb66d7e9508b1b883a602b8df122b03 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 17 Nov 2022 13:16:50 +0000
Subject: [PATCH 069/151] arm64: alternatives: make apply_alternatives_vdso()
static
We define and use apply_alternatives_vdso() within alternative.c, and
don't provide a prototype in a header. There's no need for it to be
visible outside of alternative.c, so mark it as static.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20221117131650.4056636-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/alternative.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 91263d09ea65..0f5eaa4c3a39 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -196,7 +196,7 @@ static void __apply_alternatives(const struct alt_region *region,
}
}
-void apply_alternatives_vdso(void)
+static void apply_alternatives_vdso(void)
{
struct alt_region region;
const struct elf64_hdr *hdr;
From 284d2b44a8cc0df8ed25596092fa864bc57b5bab Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Thu, 17 Nov 2022 11:41:30 +0000
Subject: [PATCH 070/151] kselftest/arm64: Use preferred form for predicate
load/stores
The preferred form of the str/ldr for predicate registers with an immediate
of zero is to omit the zero, and the clang built in assembler rejects the
zero immediate. Drop the immediate.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221117114130.687261-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/abi/syscall-abi-asm.S | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tools/testing/selftests/arm64/abi/syscall-abi-asm.S b/tools/testing/selftests/arm64/abi/syscall-abi-asm.S
index b523c21c2278..acd5e9f3bc0b 100644
--- a/tools/testing/selftests/arm64/abi/syscall-abi-asm.S
+++ b/tools/testing/selftests/arm64/abi/syscall-abi-asm.S
@@ -153,7 +153,7 @@ do_syscall:
// Only set a non-zero FFR, test patterns must be zero since the
// syscall should clear it - this lets us handle FA64.
ldr x2, =ffr_in
- ldr p0, [x2, #0]
+ ldr p0, [x2]
ldr x2, [x2, #0]
cbz x2, 2f
wrffr p0.b
@@ -298,7 +298,7 @@ do_syscall:
cbz x2, 1f
ldr x2, =ffr_out
rdffr p0.b
- str p0, [x2, #0]
+ str p0, [x2]
1:
// Restore callee saved registers x19-x30
From c8c384d7b397991584de4ed074541a54809cdd52 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 17 Nov 2022 12:09:01 +0000
Subject: [PATCH 071/151] arm64: remove current_top_of_stack()
We no longer use current_top_of_stack() on arm64, so it can be removed.
We introduced current_top_of_stack() for STACKLEAK in commit:
0b3e336601b82c6a ("arm64: Add support for STACKLEAK gcc plugin")
... then we figured out the intended semantics were unclear, and
reworked it in commit:
e85094c31ddb794a ("arm64: stackleak: fix current_top_of_stack()")
... then we removed the only user in commit:
0cfa2ccd285d98ad ("stackleak: rework stack high bound handling")
Given that it's no longer used, and it's very easy to misuse, this patch
removes current_top_of_stack(). For the moment, on_thread_stack() is
left where it is as moving it will change some header dependencies.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221117120902.3974163-2-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/processor.h | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 445aa3af3b76..cf4eefaacafc 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -396,17 +396,6 @@ long get_tagged_addr_ctrl(struct task_struct *task);
#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
#endif
-/*
- * For CONFIG_GCC_PLUGIN_STACKLEAK
- *
- * These need to be macros because otherwise we get stuck in a nightmare
- * of header definitions for the use of task_stack_page.
- */
-
-/*
- * The top of the current task's task stack
- */
-#define current_top_of_stack() ((unsigned long)current->stack + THREAD_SIZE)
#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1))
#endif /* __ASSEMBLY__ */
From 4585a934203d0ee1009bd31724190116aed01c10 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Thu, 17 Nov 2022 12:09:02 +0000
Subject: [PATCH 072/151] arm64: move on_thread_stack() to <asm/stacktrace.h>
Currently on_thread_stack() is defined in <asm/processor.h>, depending
upon definitiong from <asm/stacktrace.h> despite this header not being
included. This ends up being fragile, and any user of on_thread_stack()
must include both <asm/processor.h> and <asm/stacktrace.h>.
We organised things this way due to header dependencies back in commit:
0b3e336601b82c6a ("arm64: Add support for STACKLEAK gcc plugin")
... but now that we no longer use current_top_of_stack(), and given that
stackleak includes <asm/stacktrace.h> via <linux/stackleak.h>, we no
longer need the definition to live in <asm/processor.h>.
Move on_thread_stack() to <asm/stacktrace.h>, where all its dependencies
are guaranteed to be defined. This requires having arm64's irq.c
explicitly include <asm/stacktrace.h>, and I've taken the opportunity to
sort the includes, which were slightly out of order.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221117120902.3974163-3-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/processor.h | 2 --
arch/arm64/include/asm/stacktrace.h | 2 ++
arch/arm64/kernel/irq.c | 9 +++++----
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index cf4eefaacafc..b15fd6c11294 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -396,7 +396,5 @@ long get_tagged_addr_ctrl(struct task_struct *task);
#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
#endif
-#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1))
-
#endif /* __ASSEMBLY__ */
#endif /* __ASM_PROCESSOR_H */
diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h
index 5a0edb064ea4..4e5354beafb0 100644
--- a/arch/arm64/include/asm/stacktrace.h
+++ b/arch/arm64/include/asm/stacktrace.h
@@ -57,6 +57,8 @@ static inline bool on_task_stack(const struct task_struct *tsk,
return stackinfo_on_stack(&info, sp, size);
}
+#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1))
+
#ifdef CONFIG_VMAP_STACK
DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 38dbd3828f13..0c8e30ff8eaa 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -10,20 +10,21 @@
* Copyright (C) 2012 ARM Ltd.
*/
-#include <linux/irq.h>
-#include <linux/memory.h>
-#include <linux/smp.h>
#include <linux/hardirq.h>
#include <linux/init.h>
+#include <linux/irq.h>
#include <linux/irqchip.h>
#include <linux/kprobes.h>
+#include <linux/memory.h>
#include <linux/scs.h>
#include <linux/seq_file.h>
+#include <linux/smp.h>
#include <linux/vmalloc.h>
#include <asm/daifflags.h>
#include <asm/exception.h>
-#include <asm/vmap_stack.h>
#include <asm/softirq_stack.h>
+#include <asm/stacktrace.h>
+#include <asm/vmap_stack.h>
/* Only access this in an NMI enter/exit */
DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
From 5b468dad6e5cf4998bdc05efbc5526c111666027 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Thu, 17 Nov 2022 18:01:44 +0530
Subject: [PATCH 073/151] arm64/mm: Drop unused restore_ttbr1
restore_ttbr1 procedure is not used anywhere, hence just drop it.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221117123144.403582-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/assembler.h | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 27e0c75f7818..88175551b401 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -614,17 +614,6 @@ alternative_endif
#endif
.endm
-/*
- * Perform the reverse of offset_ttbr1.
- * bic is used as it can cover the immediate value and, in future, won't need
- * to be nop'ed out when dealing with 52-bit kernel VAs.
- */
- .macro restore_ttbr1, ttbr
-#ifdef CONFIG_ARM64_VA_BITS_52
- bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
-#endif
- .endm
-
/*
* Arrange a physical address in a TTBR register, taking care of 52-bit
* addresses.
From 07e39e60bbf0ccd5f895568e1afca032193705c0 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Wed, 16 Nov 2022 19:39:14 +0530
Subject: [PATCH 074/151] arm64: Add Cortex-715 CPU part definition
Add the CPU Partnumbers for the new Arm designs.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lore.kernel.org/r/20221116140915.356601-2-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/cputype.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index abc418650fec..4b1ad810436f 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -80,6 +80,7 @@
#define ARM_CPU_PART_CORTEX_X1 0xD44
#define ARM_CPU_PART_CORTEX_A510 0xD46
#define ARM_CPU_PART_CORTEX_A710 0xD47
+#define ARM_CPU_PART_CORTEX_A715 0xD4D
#define ARM_CPU_PART_CORTEX_X2 0xD48
#define ARM_CPU_PART_NEOVERSE_N2 0xD49
#define ARM_CPU_PART_CORTEX_A78C 0xD4B
@@ -142,6 +143,7 @@
#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
+#define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715)
#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
From 44ecda71fd8a70185c270f5914ac563827fe1d4c Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Wed, 16 Nov 2022 19:39:15 +0530
Subject: [PATCH 075/151] arm64: errata: Workaround possible Cortex-A715
[ESR|FAR]_ELx corruption
If a Cortex-A715 cpu sees a page mapping permissions change from executable
to non-executable, it may corrupt the ESR_ELx and FAR_ELx registers, on the
next instruction abort caused by permission fault.
Only user-space does executable to non-executable permission transition via
mprotect() system call which calls ptep_modify_prot_start() and ptep_modify
_prot_commit() helpers, while changing the page mapping. The platform code
can override these helpers via __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION.
Work around the problem via doing a break-before-make TLB invalidation, for
all executable user space mappings, that go through mprotect() system call.
This overrides ptep_modify_prot_start() and ptep_modify_prot_commit(), via
defining HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION on the platform thus giving
an opportunity to intercept user space exec mappings, and do the necessary
TLB invalidation. Similar interceptions are also implemented for HugeTLB.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-doc@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20221116140915.356601-3-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/silicon-errata.rst | 2 ++
arch/arm64/Kconfig | 16 ++++++++++++++++
arch/arm64/include/asm/hugetlb.h | 9 +++++++++
arch/arm64/include/asm/pgtable.h | 9 +++++++++
arch/arm64/kernel/cpu_errata.c | 7 +++++++
arch/arm64/mm/hugetlbpage.c | 21 +++++++++++++++++++++
arch/arm64/mm/mmu.c | 21 +++++++++++++++++++++
arch/arm64/tools/cpucaps | 1 +
8 files changed, 86 insertions(+)
diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst
index 808ade4cc008..ec5f889d7681 100644
--- a/Documentation/arm64/silicon-errata.rst
+++ b/Documentation/arm64/silicon-errata.rst
@@ -120,6 +120,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A710 | #2224489 | ARM64_ERRATUM_2224489 |
+----------------+-----------------+-----------------+-----------------------------+
+| ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 |
++----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 |
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 505c8a1ccbe0..56c3381e9d94 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -964,6 +964,22 @@ config ARM64_ERRATUM_2457168
If unsure, say Y.
+config ARM64_ERRATUM_2645198
+ bool "Cortex-A715: 2645198: Workaround possible [ESR|FAR]_ELx corruption"
+ default y
+ help
+ This option adds the workaround for ARM Cortex-A715 erratum 2645198.
+
+ If a Cortex-A715 cpu sees a page mapping permissions change from executable
+ to non-executable, it may corrupt the ESR_ELx and FAR_ELx registers on the
+ next instruction abort caused by permission fault.
+
+ Only user-space does executable to non-executable permission transition via
+ mprotect() system call. Workaround the problem by doing a break-before-make
+ TLB invalidation, for all changes to executable user space mappings.
+
+ If unsure, say Y.
+
config CAVIUM_ERRATUM_22375
bool "Cavium erratum 22375, 24313"
default y
diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
index d20f5da2d76f..6a4a1ab8eb23 100644
--- a/arch/arm64/include/asm/hugetlb.h
+++ b/arch/arm64/include/asm/hugetlb.h
@@ -49,6 +49,15 @@ extern pte_t huge_ptep_get(pte_t *ptep);
void __init arm64_hugetlb_cma_reserve(void);
+#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
+extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep);
+
+#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
+extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep,
+ pte_t old_pte, pte_t new_pte);
+
#include <asm-generic/hugetlb.h>
#endif /* __ASM_HUGETLB_H */
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 71a1af42f0e8..fe76e5823d91 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -1096,6 +1096,15 @@ static inline bool pud_sect_supported(void)
}
+#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
+#define ptep_modify_prot_start ptep_modify_prot_start
+extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep);
+
+#define ptep_modify_prot_commit ptep_modify_prot_commit
+extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep,
+ pte_t old_pte, pte_t new_pte);
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_PGTABLE_H */
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 89ac00084f38..307faa2b4395 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -661,6 +661,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
CAP_MIDR_RANGE_LIST(trbe_write_out_of_range_cpus),
},
#endif
+#ifdef CONFIG_ARM64_ERRATUM_2645198
+ {
+ .desc = "ARM erratum 2645198",
+ .capability = ARM64_WORKAROUND_2645198,
+ ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A715)
+ },
+#endif
#ifdef CONFIG_ARM64_ERRATUM_2077057
{
.desc = "ARM erratum 2077057",
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 35e9a468d13e..cd8d96e1fa1a 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -559,3 +559,24 @@ bool __init arch_hugetlb_valid_size(unsigned long size)
{
return __hugetlb_valid_size(size);
}
+
+pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+{
+ if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_2645198) &&
+ cpus_have_const_cap(ARM64_WORKAROUND_2645198)) {
+ /*
+ * Break-before-make (BBM) is required for all user space mappings
+ * when the permission changes from executable to non-executable
+ * in cases where cpu is affected with errata #2645198.
+ */
+ if (pte_user_exec(READ_ONCE(*ptep)))
+ return huge_ptep_clear_flush(vma, addr, ptep);
+ }
+ return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
+}
+
+void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+ pte_t old_pte, pte_t pte)
+{
+ set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
+}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9a7c38965154..5a19950e7289 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1702,3 +1702,24 @@ static int __init prevent_bootmem_remove_init(void)
}
early_initcall(prevent_bootmem_remove_init);
#endif
+
+pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+{
+ if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_2645198) &&
+ cpus_have_const_cap(ARM64_WORKAROUND_2645198)) {
+ /*
+ * Break-before-make (BBM) is required for all user space mappings
+ * when the permission changes from executable to non-executable
+ * in cases where cpu is affected with errata #2645198.
+ */
+ if (pte_user_exec(READ_ONCE(*ptep)))
+ return ptep_clear_flush(vma, addr, ptep);
+ }
+ return ptep_get_and_clear(vma->vm_mm, addr, ptep);
+}
+
+void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+ pte_t old_pte, pte_t pte)
+{
+ set_pte_at(vma->vm_mm, addr, ptep, pte);
+}
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index f1c0347ec31a..2274d836fcfe 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -70,6 +70,7 @@ WORKAROUND_2038923
WORKAROUND_2064142
WORKAROUND_2077057
WORKAROUND_2457168
+WORKAROUND_2645198
WORKAROUND_2658417
WORKAROUND_TRBE_OVERWRITE_FILL_MODE
WORKAROUND_TSB_FLUSH_FAILURE
From 32d495b0c3305546f4773b9aafcd4e90188ddb9e Mon Sep 17 00:00:00 2001
From: Will Deacon <will@kernel.org>
Date: Mon, 21 Nov 2022 12:52:07 +0000
Subject: [PATCH 076/151] Revert "arm64/mm: Drop redundant
BUG_ON(!pgtable_alloc)"
This reverts commit 9ed2b4616d4e846ece2a04cb5007ce1d1bd9e3f3.
Nathan reports early boot failures bisected to this change which look
related to the kPTI nG repainting. In any case, consolidating the
BUG_ON()s to a single location needs more thought, so revert the change
until this is figured out properly.
Link: https://lore.kernel.org/r/Y3pS5fdZ3MdLZ00t@dev-arch.thelio-3990X
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/mm/mmu.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 73a12b3abf82..d386033a074c 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -207,6 +207,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
if (flags & NO_EXEC_MAPPINGS)
pmdval |= PMD_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
pte_phys = pgtable_alloc(PAGE_SHIFT);
__pmd_populate(pmdp, pte_phys, pmdval);
pmd = READ_ONCE(*pmdp);
@@ -284,6 +285,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
if (flags & NO_EXEC_MAPPINGS)
pudval |= PUD_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
pmd_phys = pgtable_alloc(PMD_SHIFT);
__pud_populate(pudp, pmd_phys, pudval);
pud = READ_ONCE(*pudp);
@@ -322,6 +324,7 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
if (flags & NO_EXEC_MAPPINGS)
p4dval |= P4D_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
pud_phys = pgtable_alloc(PUD_SHIFT);
__p4d_populate(p4dp, pud_phys, p4dval);
p4d = READ_ONCE(*p4dp);
@@ -380,7 +383,6 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
phys &= PAGE_MASK;
addr = virt & PAGE_MASK;
end = PAGE_ALIGN(virt + size);
- BUG_ON(!pgtable_alloc);
do {
next = pgd_addr_end(addr, end);
From dcc7f001b44c30a377f44413471217cecddbee9a Mon Sep 17 00:00:00 2001
From: Shaokun Zhang <zhangshaokun@hisilicon.com>
Date: Fri, 18 Nov 2022 14:54:00 +0800
Subject: [PATCH 077/151] MAINTAINERS: Update HiSilicon PMU maintainers
Now Qi Liu has left HiSilicon and will no longer access to the
necessary hardware and document, remove the mail and thanks for
her's work.
While add the new maintainer Jonathan Cameron, He is skilled with
kernel and enough knowledge of the driver.
Cc: Jonathan Cameron <jonathan.cameron@huawei.com>
Cc: Qi Liu <liuqi6124@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Shaokun Zhang <zhangshaokun@hisilicon.com>
Acked-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Acked-by: Qi Liu <liuqi6124@gmail.com>
Link: https://lore.kernel.org/r/20221118065400.48836-1-zhangshaokun@hisilicon.com
Signed-off-by: Will Deacon <will@kernel.org>
---
MAINTAINERS | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/MAINTAINERS b/MAINTAINERS
index 046ff06ff97f..35ce60f6b8ee 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -9248,7 +9248,7 @@ F: drivers/misc/hisi_hikey_usb.c
HISILICON PMU DRIVER
M: Shaokun Zhang <zhangshaokun@hisilicon.com>
-M: Qi Liu <liuqi115@huawei.com>
+M: Jonathan Cameron <jonathan.cameron@huawei.com>
S: Supported
W: http://www.hisilicon.com
F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst
From 2016e2113d35ba06866961a39e9a9c822f2ffabd Mon Sep 17 00:00:00 2001
From: Jiucheng Xu <jiucheng.xu@amlogic.com>
Date: Mon, 21 Nov 2022 10:15:58 +0800
Subject: [PATCH 078/151] perf/amlogic: Add support for Amlogic meson G12 SoC
DDR PMU driver
Add support for Amlogic Meson G12 Series SOC - DDR bandwidth PMU driver
framework and interfaces. The PMU can not only monitor the total DDR
bandwidth, but also individual IP module bandwidth.
Signed-off-by: Jiucheng Xu <jiucheng.xu@amlogic.com>
Tested-by: Chris Healy <healych@amazon.com>
Link: https://lore.kernel.org/r/20221121021602.3306998-1-jiucheng.xu@amlogic.com
Signed-off-by: Will Deacon <will@kernel.org>
---
MAINTAINERS | 8 +
drivers/perf/Kconfig | 2 +
drivers/perf/Makefile | 1 +
drivers/perf/amlogic/Kconfig | 10 +
drivers/perf/amlogic/Makefile | 5 +
drivers/perf/amlogic/meson_ddr_pmu_core.c | 562 ++++++++++++++++++++++
drivers/perf/amlogic/meson_g12_ddr_pmu.c | 394 +++++++++++++++
include/soc/amlogic/meson_ddr_pmu.h | 66 +++
8 files changed, 1048 insertions(+)
create mode 100644 drivers/perf/amlogic/Kconfig
create mode 100644 drivers/perf/amlogic/Makefile
create mode 100644 drivers/perf/amlogic/meson_ddr_pmu_core.c
create mode 100644 drivers/perf/amlogic/meson_g12_ddr_pmu.c
create mode 100644 include/soc/amlogic/meson_ddr_pmu.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 35ce60f6b8ee..86d090ce3525 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1093,6 +1093,14 @@ S: Maintained
F: Documentation/hid/amd-sfh*
F: drivers/hid/amd-sfh-hid/
+AMLOGIC DDR PMU DRIVER
+M: Jiucheng Xu <jiucheng.xu@amlogic.com>
+L: linux-amlogic@lists.infradead.org
+S: Supported
+W: http://www.amlogic.com
+F: drivers/perf/amlogic/
+F: include/soc/amlogic/
+
AMPHION VPU CODEC V4L2 DRIVER
M: Ming Qian <ming.qian@nxp.com>
M: Shijie Qin <shijie.qin@nxp.com>
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index fa87dedf8c92..77043bcdb33c 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -201,4 +201,6 @@ config MARVELL_CN10K_DDR_PMU
source "drivers/perf/arm_cspmu/Kconfig"
+source "drivers/perf/amlogic/Kconfig"
+
endmenu
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 35bb0d979a70..13e45da61100 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -22,3 +22,4 @@ obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o
obj-$(CONFIG_APPLE_M1_CPU_PMU) += apple_m1_cpu_pmu.o
obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o
obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu/
+obj-$(CONFIG_MESON_DDR_PMU) += amlogic/
diff --git a/drivers/perf/amlogic/Kconfig b/drivers/perf/amlogic/Kconfig
new file mode 100644
index 000000000000..f68db01a7f17
--- /dev/null
+++ b/drivers/perf/amlogic/Kconfig
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config MESON_DDR_PMU
+ tristate "Amlogic DDR Bandwidth Performance Monitor"
+ depends on ARCH_MESON || COMPILE_TEST
+ help
+ Provides support for the DDR performance monitor
+ in Amlogic SoCs, which can give information about
+ memory throughput and other related events. It
+ supports multiple channels to monitor the memory
+ bandwidth simultaneously.
diff --git a/drivers/perf/amlogic/Makefile b/drivers/perf/amlogic/Makefile
new file mode 100644
index 000000000000..d3ab2ac5353b
--- /dev/null
+++ b/drivers/perf/amlogic/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_MESON_DDR_PMU) += meson_ddr_pmu_g12.o
+
+meson_ddr_pmu_g12-y := meson_ddr_pmu_core.o meson_g12_ddr_pmu.o
diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c
new file mode 100644
index 000000000000..0ff7c0449ac2
--- /dev/null
+++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/init.h>
+#include <linux/irqreturn.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/version.h>
+
+#include <soc/amlogic/meson_ddr_pmu.h>
+
+struct ddr_pmu {
+ struct pmu pmu;
+ struct dmc_info info;
+ struct dmc_counter counters; /* save counters from hw */
+ bool pmu_enabled;
+ struct device *dev;
+ char *name;
+ struct hlist_node node;
+ enum cpuhp_state cpuhp_state;
+ int cpu; /* for cpu hotplug */
+};
+
+#define DDR_PERF_DEV_NAME "meson_ddr_bw"
+#define MAX_AXI_PORTS_OF_CHANNEL 4 /* A DMC channel can monitor max 4 axi ports */
+
+#define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu)
+#define dmc_info_to_pmu(p) container_of(p, struct ddr_pmu, info)
+
+static void dmc_pmu_enable(struct ddr_pmu *pmu)
+{
+ if (!pmu->pmu_enabled)
+ pmu->info.hw_info->enable(&pmu->info);
+
+ pmu->pmu_enabled = true;
+}
+
+static void dmc_pmu_disable(struct ddr_pmu *pmu)
+{
+ if (pmu->pmu_enabled)
+ pmu->info.hw_info->disable(&pmu->info);
+
+ pmu->pmu_enabled = false;
+}
+
+static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id)
+{
+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
+ int chann;
+
+ if (event->attr.config > ALL_CHAN_COUNTER_ID &&
+ event->attr.config < COUNTER_MAX_ID) {
+ chann = event->attr.config - CHAN1_COUNTER_ID;
+
+ pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann);
+ }
+}
+
+static void ddr_cnt_addition(struct dmc_counter *sum,
+ struct dmc_counter *add1,
+ struct dmc_counter *add2,
+ int chann_nr)
+{
+ int i;
+ u64 cnt1, cnt2;
+
+ sum->all_cnt = add1->all_cnt + add2->all_cnt;
+ sum->all_req = add1->all_req + add2->all_req;
+ for (i = 0; i < chann_nr; i++) {
+ cnt1 = add1->channel_cnt[i];
+ cnt2 = add2->channel_cnt[i];
+
+ sum->channel_cnt[i] = cnt1 + cnt2;
+ }
+}
+
+static void meson_ddr_perf_event_update(struct perf_event *event)
+{
+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
+ u64 new_raw_count = 0;
+ struct dmc_counter dc = {0}, sum_dc = {0};
+ int idx;
+ int chann_nr = pmu->info.hw_info->chann_nr;
+
+ /* get the remain counters in register. */
+ pmu->info.hw_info->get_counters(&pmu->info, &dc);
+
+ ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr);
+
+ switch (event->attr.config) {
+ case ALL_CHAN_COUNTER_ID:
+ new_raw_count = sum_dc.all_cnt;
+ break;
+ case CHAN1_COUNTER_ID:
+ case CHAN2_COUNTER_ID:
+ case CHAN3_COUNTER_ID:
+ case CHAN4_COUNTER_ID:
+ case CHAN5_COUNTER_ID:
+ case CHAN6_COUNTER_ID:
+ case CHAN7_COUNTER_ID:
+ case CHAN8_COUNTER_ID:
+ idx = event->attr.config - CHAN1_COUNTER_ID;
+ new_raw_count = sum_dc.channel_cnt[idx];
+ break;
+ }
+
+ local64_set(&event->count, new_raw_count);
+}
+
+static int meson_ddr_perf_event_init(struct perf_event *event)
+{
+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
+ u64 config1 = event->attr.config1;
+ u64 config2 = event->attr.config2;
+
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+ return -EOPNOTSUPP;
+
+ if (event->cpu < 0)
+ return -EOPNOTSUPP;
+
+ /* check if the number of parameters is too much */
+ if (event->attr.config != ALL_CHAN_COUNTER_ID &&
+ hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL)
+ return -EOPNOTSUPP;
+
+ event->cpu = pmu->cpu;
+
+ return 0;
+}
+
+static void meson_ddr_perf_event_start(struct perf_event *event, int flags)
+{
+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
+
+ memset(&pmu->counters, 0, sizeof(pmu->counters));
+ dmc_pmu_enable(pmu);
+}
+
+static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
+{
+ u64 config1 = event->attr.config1;
+ u64 config2 = event->attr.config2;
+ int i;
+
+ for_each_set_bit(i, (const unsigned long *)&config1, sizeof(config1))
+ meson_ddr_set_axi_filter(event, i);
+
+ for_each_set_bit(i, (const unsigned long *)&config2, sizeof(config2))
+ meson_ddr_set_axi_filter(event, i + 64);
+
+ if (flags & PERF_EF_START)
+ meson_ddr_perf_event_start(event, flags);
+
+ return 0;
+}
+
+static void meson_ddr_perf_event_stop(struct perf_event *event, int flags)
+{
+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
+
+ if (flags & PERF_EF_UPDATE)
+ meson_ddr_perf_event_update(event);
+
+ dmc_pmu_disable(pmu);
+}
+
+static void meson_ddr_perf_event_del(struct perf_event *event, int flags)
+{
+ meson_ddr_perf_event_stop(event, PERF_EF_UPDATE);
+}
+
+static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
+
+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+}
+
+static struct device_attribute meson_ddr_perf_cpumask_attr =
+__ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL);
+
+static struct attribute *meson_ddr_perf_cpumask_attrs[] = {
+ &meson_ddr_perf_cpumask_attr.attr,
+ NULL,
+};
+
+static const struct attribute_group ddr_perf_cpumask_attr_group = {
+ .attrs = meson_ddr_perf_cpumask_attrs,
+};
+
+static ssize_t
+pmu_event_show(struct device *dev, struct device_attribute *attr,
+ char *page)
+{
+ struct perf_pmu_events_attr *pmu_attr;
+
+ pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
+ return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
+}
+
+static ssize_t
+event_show_unit(struct device *dev, struct device_attribute *attr,
+ char *page)
+{
+ return sysfs_emit(page, "MB\n");
+}
+
+static ssize_t
+event_show_scale(struct device *dev, struct device_attribute *attr,
+ char *page)
+{
+ /* one count = 16byte = 1.52587890625e-05 MB */
+ return sysfs_emit(page, "1.52587890625e-05\n");
+}
+
+#define AML_DDR_PMU_EVENT_ATTR(_name, _id) \
+{ \
+ .attr = __ATTR(_name, 0444, pmu_event_show, NULL), \
+ .id = _id, \
+}
+
+#define AML_DDR_PMU_EVENT_UNIT_ATTR(_name) \
+ __ATTR(_name.unit, 0444, event_show_unit, NULL)
+
+#define AML_DDR_PMU_EVENT_SCALE_ATTR(_name) \
+ __ATTR(_name.scale, 0444, event_show_scale, NULL)
+
+static struct device_attribute event_unit_attrs[] = {
+ AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes),
+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes),
+};
+
+static struct device_attribute event_scale_attrs[] = {
+ AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes),
+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes),
+};
+
+static struct perf_pmu_events_attr event_attrs[] = {
+ AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID),
+ AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID),
+};
+
+/* three attrs are combined an event */
+static struct attribute *ddr_perf_events_attrs[COUNTER_MAX_ID * 3];
+
+static struct attribute_group ddr_perf_events_attr_group = {
+ .name = "events",
+ .attrs = ddr_perf_events_attrs,
+};
+
+static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj,
+ struct attribute *attr,
+ int n)
+{
+ struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
+ struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
+ const u64 *capability = ddr_pmu->info.hw_info->capability;
+ struct device_attribute *dev_attr;
+ int id;
+ char value[20]; // config1:xxx, 20 is enough
+
+ dev_attr = container_of(attr, struct device_attribute, attr);
+ dev_attr->show(NULL, NULL, value);
+
+ if (sscanf(value, "config1:%d", &id) == 1)
+ return capability[0] & (1ULL << id) ? attr->mode : 0;
+
+ if (sscanf(value, "config2:%d", &id) == 1)
+ return capability[1] & (1ULL << id) ? attr->mode : 0;
+
+ return attr->mode;
+}
+
+static struct attribute_group ddr_perf_format_attr_group = {
+ .name = "format",
+ .is_visible = meson_ddr_perf_format_attr_visible,
+};
+
+static ssize_t meson_ddr_perf_identifier_show(struct device *dev,
+ struct device_attribute *attr,
+ char *page)
+{
+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
+
+ return sysfs_emit(page, "%s\n", pmu->name);
+}
+
+static struct device_attribute meson_ddr_perf_identifier_attr =
+__ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL);
+
+static struct attribute *meson_ddr_perf_identifier_attrs[] = {
+ &meson_ddr_perf_identifier_attr.attr,
+ NULL,
+};
+
+static const struct attribute_group ddr_perf_identifier_attr_group = {
+ .attrs = meson_ddr_perf_identifier_attrs,
+};
+
+static const struct attribute_group *attr_groups[] = {
+ &ddr_perf_events_attr_group,
+ &ddr_perf_format_attr_group,
+ &ddr_perf_cpumask_attr_group,
+ &ddr_perf_identifier_attr_group,
+ NULL,
+};
+
+static irqreturn_t dmc_irq_handler(int irq, void *dev_id)
+{
+ struct dmc_info *info = dev_id;
+ struct ddr_pmu *pmu;
+ struct dmc_counter counters, *sum_cnter;
+ int i;
+
+ pmu = dmc_info_to_pmu(info);
+
+ if (info->hw_info->irq_handler(info, &counters) != 0)
+ goto out;
+
+ sum_cnter = &pmu->counters;
+ sum_cnter->all_cnt += counters.all_cnt;
+ sum_cnter->all_req += counters.all_req;
+
+ for (i = 0; i < pmu->info.hw_info->chann_nr; i++)
+ sum_cnter->channel_cnt[i] += counters.channel_cnt[i];
+
+ if (pmu->pmu_enabled)
+ /*
+ * the timer interrupt only supprt
+ * one shot mode, we have to re-enable
+ * it in ISR to support continue mode.
+ */
+ info->hw_info->enable(info);
+
+ dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t"
+ "sum: %llu %llu %llu, %llu, %llu, %llu\n",
+ counters.all_req,
+ counters.all_cnt,
+ counters.channel_cnt[0],
+ counters.channel_cnt[1],
+ counters.channel_cnt[2],
+ counters.channel_cnt[3],
+
+ pmu->counters.all_req,
+ pmu->counters.all_cnt,
+ pmu->counters.channel_cnt[0],
+ pmu->counters.channel_cnt[1],
+ pmu->counters.channel_cnt[2],
+ pmu->counters.channel_cnt[3]);
+out:
+ return IRQ_HANDLED;
+}
+
+static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
+ int target;
+
+ if (cpu != pmu->cpu)
+ return 0;
+
+ target = cpumask_any_but(cpu_online_mask, cpu);
+ if (target >= nr_cpu_ids)
+ return 0;
+
+ perf_pmu_migrate_context(&pmu->pmu, cpu, target);
+ pmu->cpu = target;
+
+ WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu)));
+
+ return 0;
+}
+
+static void fill_event_attr(struct ddr_pmu *pmu)
+{
+ int i, j, k;
+ struct attribute **dst = ddr_perf_events_attrs;
+
+ j = 0;
+ k = 0;
+
+ /* fill ALL_CHAN_COUNTER_ID event */
+ dst[j++] = &event_attrs[k].attr.attr;
+ dst[j++] = &event_unit_attrs[k].attr;
+ dst[j++] = &event_scale_attrs[k].attr;
+
+ k++;
+
+ /* fill each channel event */
+ for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) {
+ dst[j++] = &event_attrs[k].attr.attr;
+ dst[j++] = &event_unit_attrs[k].attr;
+ dst[j++] = &event_scale_attrs[k].attr;
+ }
+
+ dst[j] = NULL; /* mark end */
+}
+
+static void fmt_attr_fill(struct attribute **fmt_attr)
+{
+ ddr_perf_format_attr_group.attrs = fmt_attr;
+}
+
+static int ddr_pmu_parse_dt(struct platform_device *pdev,
+ struct dmc_info *info)
+{
+ void __iomem *base;
+ int i, ret;
+
+ info->hw_info = of_device_get_match_data(&pdev->dev);
+
+ for (i = 0; i < info->hw_info->dmc_nr; i++) {
+ /* resource 0 for ddr register base */
+ base = devm_platform_ioremap_resource(pdev, i);
+ if (IS_ERR(base))
+ return PTR_ERR(base);
+
+ info->ddr_reg[i] = base;
+ }
+
+ /* resource i for pll register base */
+ base = devm_platform_ioremap_resource(pdev, i);
+ if (IS_ERR(base))
+ return PTR_ERR(base);
+
+ info->pll_reg = base;
+
+ ret = platform_get_irq(pdev, 0);
+ if (ret < 0)
+ return ret;
+
+ info->irq_num = ret;
+
+ ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler,
+ IRQF_NOBALANCING, dev_name(&pdev->dev),
+ (void *)info);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+int meson_ddr_pmu_create(struct platform_device *pdev)
+{
+ int ret;
+ char *name;
+ struct ddr_pmu *pmu;
+
+ pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL);
+ if (!pmu)
+ return -ENOMEM;
+
+ *pmu = (struct ddr_pmu) {
+ .pmu = {
+ .module = THIS_MODULE,
+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
+ .task_ctx_nr = perf_invalid_context,
+ .attr_groups = attr_groups,
+ .event_init = meson_ddr_perf_event_init,
+ .add = meson_ddr_perf_event_add,
+ .del = meson_ddr_perf_event_del,
+ .start = meson_ddr_perf_event_start,
+ .stop = meson_ddr_perf_event_stop,
+ .read = meson_ddr_perf_event_update,
+ },
+ };
+
+ ret = ddr_pmu_parse_dt(pdev, &pmu->info);
+ if (ret < 0)
+ return ret;
+
+ fmt_attr_fill(pmu->info.hw_info->fmt_attr);
+
+ pmu->cpu = smp_processor_id();
+
+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME);
+ if (!name)
+ return -ENOMEM;
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL,
+ ddr_perf_offline_cpu);
+ if (ret < 0)
+ return ret;
+
+ pmu->cpuhp_state = ret;
+
+ /* Register the pmu instance for cpu hotplug */
+ ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
+ if (ret)
+ goto cpuhp_instance_err;
+
+ fill_event_attr(pmu);
+
+ ret = perf_pmu_register(&pmu->pmu, name, -1);
+ if (ret)
+ goto pmu_register_err;
+
+ pmu->name = name;
+ pmu->dev = &pdev->dev;
+ pmu->pmu_enabled = false;
+
+ platform_set_drvdata(pdev, pmu);
+
+ return 0;
+
+pmu_register_err:
+ cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
+
+cpuhp_instance_err:
+ cpuhp_remove_state(pmu->cpuhp_state);
+
+ return ret;
+}
+
+int meson_ddr_pmu_remove(struct platform_device *pdev)
+{
+ struct ddr_pmu *pmu = platform_get_drvdata(pdev);
+
+ perf_pmu_unregister(&pmu->pmu);
+ cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
+ cpuhp_remove_state(pmu->cpuhp_state);
+
+ return 0;
+}
diff --git a/drivers/perf/amlogic/meson_g12_ddr_pmu.c b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
new file mode 100644
index 000000000000..c07c34f03cce
--- /dev/null
+++ b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
@@ -0,0 +1,394 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
+ */
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/types.h>
+#include <linux/version.h>
+
+#include <soc/amlogic/meson_ddr_pmu.h>
+
+#define PORT_MAJOR 32
+#define DEFAULT_XTAL_FREQ 24000000UL
+
+#define DMC_QOS_IRQ BIT(30)
+
+/* DMC bandwidth monitor register address offset */
+#define DMC_MON_G12_CTRL0 (0x20 << 2)
+#define DMC_MON_G12_CTRL1 (0x21 << 2)
+#define DMC_MON_G12_CTRL2 (0x22 << 2)
+#define DMC_MON_G12_CTRL3 (0x23 << 2)
+#define DMC_MON_G12_CTRL4 (0x24 << 2)
+#define DMC_MON_G12_CTRL5 (0x25 << 2)
+#define DMC_MON_G12_CTRL6 (0x26 << 2)
+#define DMC_MON_G12_CTRL7 (0x27 << 2)
+#define DMC_MON_G12_CTRL8 (0x28 << 2)
+
+#define DMC_MON_G12_ALL_REQ_CNT (0x29 << 2)
+#define DMC_MON_G12_ALL_GRANT_CNT (0x2a << 2)
+#define DMC_MON_G12_ONE_GRANT_CNT (0x2b << 2)
+#define DMC_MON_G12_SEC_GRANT_CNT (0x2c << 2)
+#define DMC_MON_G12_THD_GRANT_CNT (0x2d << 2)
+#define DMC_MON_G12_FOR_GRANT_CNT (0x2e << 2)
+#define DMC_MON_G12_TIMER (0x2f << 2)
+
+/* Each bit represent a axi line */
+PMU_FORMAT_ATTR(event, "config:0-7");
+PMU_FORMAT_ATTR(arm, "config1:0");
+PMU_FORMAT_ATTR(gpu, "config1:1");
+PMU_FORMAT_ATTR(pcie, "config1:2");
+PMU_FORMAT_ATTR(hdcp, "config1:3");
+PMU_FORMAT_ATTR(hevc_front, "config1:4");
+PMU_FORMAT_ATTR(usb3_0, "config1:6");
+PMU_FORMAT_ATTR(device, "config1:7");
+PMU_FORMAT_ATTR(hevc_back, "config1:8");
+PMU_FORMAT_ATTR(h265enc, "config1:9");
+PMU_FORMAT_ATTR(vpu_read1, "config1:16");
+PMU_FORMAT_ATTR(vpu_read2, "config1:17");
+PMU_FORMAT_ATTR(vpu_read3, "config1:18");
+PMU_FORMAT_ATTR(vpu_write1, "config1:19");
+PMU_FORMAT_ATTR(vpu_write2, "config1:20");
+PMU_FORMAT_ATTR(vdec, "config1:21");
+PMU_FORMAT_ATTR(hcodec, "config1:22");
+PMU_FORMAT_ATTR(ge2d, "config1:23");
+
+PMU_FORMAT_ATTR(spicc1, "config1:32");
+PMU_FORMAT_ATTR(usb0, "config1:33");
+PMU_FORMAT_ATTR(dma, "config1:34");
+PMU_FORMAT_ATTR(arb0, "config1:35");
+PMU_FORMAT_ATTR(sd_emmc_b, "config1:36");
+PMU_FORMAT_ATTR(usb1, "config1:37");
+PMU_FORMAT_ATTR(audio, "config1:38");
+PMU_FORMAT_ATTR(aififo, "config1:39");
+PMU_FORMAT_ATTR(parser, "config1:41");
+PMU_FORMAT_ATTR(ao_cpu, "config1:42");
+PMU_FORMAT_ATTR(sd_emmc_c, "config1:43");
+PMU_FORMAT_ATTR(spicc2, "config1:44");
+PMU_FORMAT_ATTR(ethernet, "config1:45");
+PMU_FORMAT_ATTR(sana, "config1:46");
+
+/* for sm1 and g12b */
+PMU_FORMAT_ATTR(nna, "config1:10");
+
+/* for g12b only */
+PMU_FORMAT_ATTR(gdc, "config1:11");
+PMU_FORMAT_ATTR(mipi_isp, "config1:12");
+PMU_FORMAT_ATTR(arm1, "config1:13");
+PMU_FORMAT_ATTR(sd_emmc_a, "config1:40");
+
+static struct attribute *g12_pmu_format_attrs[] = {
+ &format_attr_event.attr,
+ &format_attr_arm.attr,
+ &format_attr_gpu.attr,
+ &format_attr_nna.attr,
+ &format_attr_gdc.attr,
+ &format_attr_arm1.attr,
+ &format_attr_mipi_isp.attr,
+ &format_attr_sd_emmc_a.attr,
+ &format_attr_pcie.attr,
+ &format_attr_hdcp.attr,
+ &format_attr_hevc_front.attr,
+ &format_attr_usb3_0.attr,
+ &format_attr_device.attr,
+ &format_attr_hevc_back.attr,
+ &format_attr_h265enc.attr,
+ &format_attr_vpu_read1.attr,
+ &format_attr_vpu_read2.attr,
+ &format_attr_vpu_read3.attr,
+ &format_attr_vpu_write1.attr,
+ &format_attr_vpu_write2.attr,
+ &format_attr_vdec.attr,
+ &format_attr_hcodec.attr,
+ &format_attr_ge2d.attr,
+ &format_attr_spicc1.attr,
+ &format_attr_usb0.attr,
+ &format_attr_dma.attr,
+ &format_attr_arb0.attr,
+ &format_attr_sd_emmc_b.attr,
+ &format_attr_usb1.attr,
+ &format_attr_audio.attr,
+ &format_attr_aififo.attr,
+ &format_attr_parser.attr,
+ &format_attr_ao_cpu.attr,
+ &format_attr_sd_emmc_c.attr,
+ &format_attr_spicc2.attr,
+ &format_attr_ethernet.attr,
+ &format_attr_sana.attr,
+ NULL,
+};
+
+/* calculate ddr clock */
+static unsigned long dmc_g12_get_freq_quick(struct dmc_info *info)
+{
+ unsigned int val;
+ unsigned int n, m, od1;
+ unsigned int od_div = 0xfff;
+ unsigned long freq = 0;
+
+ val = readl(info->pll_reg);
+ val = val & 0xfffff;
+ switch ((val >> 16) & 7) {
+ case 0:
+ od_div = 2;
+ break;
+
+ case 1:
+ od_div = 3;
+ break;
+
+ case 2:
+ od_div = 4;
+ break;
+
+ case 3:
+ od_div = 6;
+ break;
+
+ case 4:
+ od_div = 8;
+ break;
+
+ default:
+ break;
+ }
+
+ m = val & 0x1ff;
+ n = ((val >> 10) & 0x1f);
+ od1 = (((val >> 19) & 0x1)) == 1 ? 2 : 1;
+ freq = DEFAULT_XTAL_FREQ / 1000; /* avoid overflow */
+ if (n)
+ freq = ((((freq * m) / n) >> od1) / od_div) * 1000;
+
+ return freq;
+}
+
+#ifdef DEBUG
+static void g12_dump_reg(struct dmc_info *db)
+{
+ int s = 0, i;
+ unsigned int r;
+
+ for (i = 0; i < 9; i++) {
+ r = readl(db->ddr_reg[0] + (DMC_MON_G12_CTRL0 + (i << 2)));
+ pr_notice("DMC_MON_CTRL%d: %08x\n", i, r);
+ }
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
+ pr_notice("DMC_MON_ALL_REQ_CNT: %08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
+ pr_notice("DMC_MON_ALL_GRANT_CNT:%08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ONE_GRANT_CNT);
+ pr_notice("DMC_MON_ONE_GRANT_CNT:%08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_SEC_GRANT_CNT);
+ pr_notice("DMC_MON_SEC_GRANT_CNT:%08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_THD_GRANT_CNT);
+ pr_notice("DMC_MON_THD_GRANT_CNT:%08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_FOR_GRANT_CNT);
+ pr_notice("DMC_MON_FOR_GRANT_CNT:%08x\n", r);
+ r = readl(db->ddr_reg[0] + DMC_MON_G12_TIMER);
+ pr_notice("DMC_MON_TIMER: %08x\n", r);
+}
+#endif
+
+static void dmc_g12_counter_enable(struct dmc_info *info)
+{
+ unsigned int val;
+ unsigned long clock_count = dmc_g12_get_freq_quick(info) / 10; /* 100ms */
+
+ writel(clock_count, info->ddr_reg[0] + DMC_MON_G12_TIMER);
+
+ val = readl(info->ddr_reg[0] + DMC_MON_G12_CTRL0);
+
+ /* enable all channel */
+ val = BIT(31) | /* enable bit */
+ BIT(20) | /* use timer */
+ 0x0f; /* 4 channels */
+
+ writel(val, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
+
+#ifdef DEBUG
+ g12_dump_reg(info);
+#endif
+}
+
+static void dmc_g12_config_fiter(struct dmc_info *info,
+ int port, int channel)
+{
+ u32 val;
+ u32 rp[MAX_CHANNEL_NUM] = {DMC_MON_G12_CTRL1, DMC_MON_G12_CTRL3,
+ DMC_MON_G12_CTRL5, DMC_MON_G12_CTRL7};
+ u32 rs[MAX_CHANNEL_NUM] = {DMC_MON_G12_CTRL2, DMC_MON_G12_CTRL4,
+ DMC_MON_G12_CTRL6, DMC_MON_G12_CTRL8};
+ int subport = -1;
+
+ /* clear all port mask */
+ if (port < 0) {
+ writel(0, info->ddr_reg[0] + rp[channel]);
+ writel(0, info->ddr_reg[0] + rs[channel]);
+ return;
+ }
+
+ if (port >= PORT_MAJOR)
+ subport = port - PORT_MAJOR;
+
+ if (subport < 0) {
+ val = readl(info->ddr_reg[0] + rp[channel]);
+ val |= (1 << port);
+ writel(val, info->ddr_reg[0] + rp[channel]);
+ val = 0xffff;
+ writel(val, info->ddr_reg[0] + rs[channel]);
+ } else {
+ val = BIT(23); /* select device */
+ writel(val, info->ddr_reg[0] + rp[channel]);
+ val = readl(info->ddr_reg[0] + rs[channel]);
+ val |= (1 << subport);
+ writel(val, info->ddr_reg[0] + rs[channel]);
+ }
+}
+
+static void dmc_g12_set_axi_filter(struct dmc_info *info, int axi_id, int channel)
+{
+ if (channel > info->hw_info->chann_nr)
+ return;
+
+ dmc_g12_config_fiter(info, axi_id, channel);
+}
+
+static void dmc_g12_counter_disable(struct dmc_info *info)
+{
+ int i;
+
+ /* clear timer */
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_TIMER);
+
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ONE_GRANT_CNT);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_SEC_GRANT_CNT);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_THD_GRANT_CNT);
+ writel(0, info->ddr_reg[0] + DMC_MON_G12_FOR_GRANT_CNT);
+
+ /* clear port channel mapping */
+ for (i = 0; i < info->hw_info->chann_nr; i++)
+ dmc_g12_config_fiter(info, -1, i);
+}
+
+static void dmc_g12_get_counters(struct dmc_info *info,
+ struct dmc_counter *counter)
+{
+ int i;
+ unsigned int reg;
+
+ counter->all_cnt = readl(info->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
+ counter->all_req = readl(info->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
+
+ for (i = 0; i < info->hw_info->chann_nr; i++) {
+ reg = DMC_MON_G12_ONE_GRANT_CNT + (i << 2);
+ counter->channel_cnt[i] = readl(info->ddr_reg[0] + reg);
+ }
+}
+
+static int dmc_g12_irq_handler(struct dmc_info *info,
+ struct dmc_counter *counter)
+{
+ unsigned int val;
+ int ret = -EINVAL;
+
+ val = readl(info->ddr_reg[0] + DMC_MON_G12_CTRL0);
+ if (val & DMC_QOS_IRQ) {
+ dmc_g12_get_counters(info, counter);
+ /* clear irq flags */
+ writel(val, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
+ ret = 0;
+ }
+ return ret;
+}
+
+static const struct dmc_hw_info g12a_dmc_info = {
+ .enable = dmc_g12_counter_enable,
+ .disable = dmc_g12_counter_disable,
+ .irq_handler = dmc_g12_irq_handler,
+ .get_counters = dmc_g12_get_counters,
+ .set_axi_filter = dmc_g12_set_axi_filter,
+
+ .dmc_nr = 1,
+ .chann_nr = 4,
+ .capability = {0X7EFF00FF03DF, 0},
+ .fmt_attr = g12_pmu_format_attrs,
+};
+
+static const struct dmc_hw_info g12b_dmc_info = {
+ .enable = dmc_g12_counter_enable,
+ .disable = dmc_g12_counter_disable,
+ .irq_handler = dmc_g12_irq_handler,
+ .get_counters = dmc_g12_get_counters,
+ .set_axi_filter = dmc_g12_set_axi_filter,
+
+ .dmc_nr = 1,
+ .chann_nr = 4,
+ .capability = {0X7FFF00FF3FDF, 0},
+ .fmt_attr = g12_pmu_format_attrs,
+};
+
+static const struct dmc_hw_info sm1_dmc_info = {
+ .enable = dmc_g12_counter_enable,
+ .disable = dmc_g12_counter_disable,
+ .irq_handler = dmc_g12_irq_handler,
+ .get_counters = dmc_g12_get_counters,
+ .set_axi_filter = dmc_g12_set_axi_filter,
+
+ .dmc_nr = 1,
+ .chann_nr = 4,
+ .capability = {0X7EFF00FF07DF, 0},
+ .fmt_attr = g12_pmu_format_attrs,
+};
+
+static int g12_ddr_pmu_probe(struct platform_device *pdev)
+{
+ return meson_ddr_pmu_create(pdev);
+}
+
+static int g12_ddr_pmu_remove(struct platform_device *pdev)
+{
+ meson_ddr_pmu_remove(pdev);
+
+ return 0;
+}
+
+static const struct of_device_id meson_ddr_pmu_dt_match[] = {
+ {
+ .compatible = "amlogic,g12a-ddr-pmu",
+ .data = &g12a_dmc_info,
+ },
+ {
+ .compatible = "amlogic,g12b-ddr-pmu",
+ .data = &g12b_dmc_info,
+ },
+ {
+ .compatible = "amlogic,sm1-ddr-pmu",
+ .data = &sm1_dmc_info,
+ },
+ {}
+};
+
+static struct platform_driver g12_ddr_pmu_driver = {
+ .probe = g12_ddr_pmu_probe,
+ .remove = g12_ddr_pmu_remove,
+
+ .driver = {
+ .name = "meson-g12-ddr-pmu",
+ .of_match_table = meson_ddr_pmu_dt_match,
+ },
+};
+
+module_platform_driver(g12_ddr_pmu_driver);
+MODULE_AUTHOR("Jiucheng Xu");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Amlogic G12 series SoC DDR PMU");
diff --git a/include/soc/amlogic/meson_ddr_pmu.h b/include/soc/amlogic/meson_ddr_pmu.h
new file mode 100644
index 000000000000..4a33e4ab8ada
--- /dev/null
+++ b/include/soc/amlogic/meson_ddr_pmu.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
+ */
+
+#ifndef __MESON_DDR_PMU_H__
+#define __MESON_DDR_PMU_H__
+
+#define MAX_CHANNEL_NUM 8
+
+enum {
+ ALL_CHAN_COUNTER_ID,
+ CHAN1_COUNTER_ID,
+ CHAN2_COUNTER_ID,
+ CHAN3_COUNTER_ID,
+ CHAN4_COUNTER_ID,
+ CHAN5_COUNTER_ID,
+ CHAN6_COUNTER_ID,
+ CHAN7_COUNTER_ID,
+ CHAN8_COUNTER_ID,
+ COUNTER_MAX_ID,
+};
+
+struct dmc_info;
+
+struct dmc_counter {
+ u64 all_cnt; /* The count of all requests come in/out ddr controller */
+ union {
+ u64 all_req;
+ struct {
+ u64 all_idle_cnt;
+ u64 all_16bit_cnt;
+ };
+ };
+ u64 channel_cnt[MAX_CHANNEL_NUM]; /* To save a DMC bandwidth-monitor channel counter */
+};
+
+struct dmc_hw_info {
+ void (*enable)(struct dmc_info *info);
+ void (*disable)(struct dmc_info *info);
+ /* Bind an axi line to a bandwidth-monitor channel */
+ void (*set_axi_filter)(struct dmc_info *info, int axi_id, int chann);
+ int (*irq_handler)(struct dmc_info *info,
+ struct dmc_counter *counter);
+ void (*get_counters)(struct dmc_info *info,
+ struct dmc_counter *counter);
+
+ int dmc_nr; /* The number of dmc controller */
+ int chann_nr; /* The number of dmc bandwidth monitor channels */
+ struct attribute **fmt_attr;
+ const u64 capability[2];
+};
+
+struct dmc_info {
+ const struct dmc_hw_info *hw_info;
+
+ void __iomem *ddr_reg[4];
+ unsigned long timer_value; /* Timer value in TIMER register */
+ void __iomem *pll_reg;
+ int irq_num; /* irq vector number */
+};
+
+int meson_ddr_pmu_create(struct platform_device *pdev);
+int meson_ddr_pmu_remove(struct platform_device *pdev);
+
+#endif /* __MESON_DDR_PMU_H__ */
From 537216e59f0c126670749f8dc09f5c4f03375234 Mon Sep 17 00:00:00 2001
From: Jiucheng Xu <jiucheng.xu@amlogic.com>
Date: Mon, 21 Nov 2022 10:15:59 +0800
Subject: [PATCH 079/151] docs/perf: Add documentation for the Amlogic G12 DDR
PMU
Add a user guide to show how to use DDR PMU to
monitor DDR bandwidth on Amlogic G12 SoC
Signed-off-by: Jiucheng Xu <jiucheng.xu@amlogic.com>
Reviewed-by: Chris Healy <healych@amazon.com>
Link: https://lore.kernel.org/r/20221121021602.3306998-2-jiucheng.xu@amlogic.com
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/admin-guide/perf/index.rst | 1 +
.../admin-guide/perf/meson-ddr-pmu.rst | 70 +++++++++++++++++++
MAINTAINERS | 1 +
3 files changed, 72 insertions(+)
create mode 100644 Documentation/admin-guide/perf/meson-ddr-pmu.rst
diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst
index 7ed239408138..9de64a40adab 100644
--- a/Documentation/admin-guide/perf/index.rst
+++ b/Documentation/admin-guide/perf/index.rst
@@ -20,3 +20,4 @@ Performance monitor support
thunderx2-pmu
alibaba_pmu
nvidia-pmu
+ meson-ddr-pmu
diff --git a/Documentation/admin-guide/perf/meson-ddr-pmu.rst b/Documentation/admin-guide/perf/meson-ddr-pmu.rst
new file mode 100644
index 000000000000..8e71be1d6346
--- /dev/null
+++ b/Documentation/admin-guide/perf/meson-ddr-pmu.rst
@@ -0,0 +1,70 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===========================================================
+Amlogic SoC DDR Bandwidth Performance Monitoring Unit (PMU)
+===========================================================
+
+The Amlogic Meson G12 SoC contains a bandwidth monitor inside DRAM controller.
+The monitor includes 4 channels. Each channel can count the request accessing
+DRAM. The channel can count up to 3 AXI port simultaneously. It can be helpful
+to show if the performance bottleneck is on DDR bandwidth.
+
+Currently, this driver supports the following 5 perf events:
+
++ meson_ddr_bw/total_rw_bytes/
++ meson_ddr_bw/chan_1_rw_bytes/
++ meson_ddr_bw/chan_2_rw_bytes/
++ meson_ddr_bw/chan_3_rw_bytes/
++ meson_ddr_bw/chan_4_rw_bytes/
+
+meson_ddr_bw/chan_{1,2,3,4}_rw_bytes/ events are channel-specific events.
+Each channel support filtering, which can let the channel to monitor
+individual IP module in SoC.
+
+Below are DDR access request event filter keywords:
+
++ arm - from CPU
++ vpu_read1 - from OSD + VPP read
++ gpu - from 3D GPU
++ pcie - from PCIe controller
++ hdcp - from HDCP controller
++ hevc_front - from HEVC codec front end
++ usb3_0 - from USB3.0 controller
++ hevc_back - from HEVC codec back end
++ h265enc - from HEVC encoder
++ vpu_read2 - from DI read
++ vpu_write1 - from VDIN write
++ vpu_write2 - from di write
++ vdec - from legacy codec video decoder
++ hcodec - from H264 encoder
++ ge2d - from ge2d
++ spicc1 - from SPI controller 1
++ usb0 - from USB2.0 controller 0
++ dma - from system DMA controller 1
++ arb0 - from arb0
++ sd_emmc_b - from SD eMMC b controller
++ usb1 - from USB2.0 controller 1
++ audio - from Audio module
++ sd_emmc_c - from SD eMMC c controller
++ spicc2 - from SPI controller 2
++ ethernet - from Ethernet controller
+
+
+Examples:
+
+ + Show the total DDR bandwidth per seconds:
+
+ .. code-block:: bash
+
+ perf stat -a -e meson_ddr_bw/total_rw_bytes/ -I 1000 sleep 10
+
+
+ + Show individual DDR bandwidth from CPU and GPU respectively, as well as
+ sum of them:
+
+ .. code-block:: bash
+
+ perf stat -a -e meson_ddr_bw/chan_1_rw_bytes,arm=1/ -I 1000 sleep 10
+ perf stat -a -e meson_ddr_bw/chan_2_rw_bytes,gpu=1/ -I 1000 sleep 10
+ perf stat -a -e meson_ddr_bw/chan_3_rw_bytes,arm=1,gpu=1/ -I 1000 sleep 10
+
diff --git a/MAINTAINERS b/MAINTAINERS
index 86d090ce3525..1157873ccc99 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1098,6 +1098,7 @@ M: Jiucheng Xu <jiucheng.xu@amlogic.com>
L: linux-amlogic@lists.infradead.org
S: Supported
W: http://www.amlogic.com
+F: Documentation/admin-guide/perf/meson-ddr-pmu.rst
F: drivers/perf/amlogic/
F: include/soc/amlogic/
From fd9678d3beaa5c78dfd878f67bc543f1702cc9d9 Mon Sep 17 00:00:00 2001
From: Jiucheng Xu <jiucheng.xu@amlogic.com>
Date: Mon, 21 Nov 2022 10:16:00 +0800
Subject: [PATCH 080/151] dt-binding: perf: Add Amlogic DDR PMU
Add binding documentation for the Amlogic G12 series DDR
performance monitor unit.
Signed-off-by: Jiucheng Xu <jiucheng.xu@amlogic.com>
Reviewed-by: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
Link: https://lore.kernel.org/r/20221121021602.3306998-3-jiucheng.xu@amlogic.com
Signed-off-by: Will Deacon <will@kernel.org>
---
.../bindings/perf/amlogic,g12-ddr-pmu.yaml | 54 +++++++++++++++++++
MAINTAINERS | 1 +
2 files changed, 55 insertions(+)
create mode 100644 Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml
diff --git a/Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml b/Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml
new file mode 100644
index 000000000000..50f46a6898b1
--- /dev/null
+++ b/Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/perf/amlogic,g12-ddr-pmu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Amlogic G12 DDR performance monitor
+
+maintainers:
+ - Jiucheng Xu <jiucheng.xu@amlogic.com>
+
+description: |
+ Amlogic G12 series SoC integrate DDR bandwidth monitor.
+ A timer is inside and can generate interrupt when timeout.
+ The bandwidth is counted in the timer ISR. Different platform
+ has different subset of event format attribute.
+
+properties:
+ compatible:
+ enum:
+ - amlogic,g12a-ddr-pmu
+ - amlogic,g12b-ddr-pmu
+ - amlogic,sm1-ddr-pmu
+
+ reg:
+ items:
+ - description: DMC bandwidth register space.
+ - description: DMC PLL register space.
+
+ interrupts:
+ items:
+ - description: The IRQ of the inside timer timeout.
+
+required:
+ - compatible
+ - reg
+ - interrupts
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pmu {
+ #address-cells=<2>;
+ #size-cells=<2>;
+
+ pmu@ff638000 {
+ compatible = "amlogic,g12a-ddr-pmu";
+ reg = <0x0 0xff638000 0x0 0x100>,
+ <0x0 0xff638c00 0x0 0x100>;
+ interrupts = <GIC_SPI 52 IRQ_TYPE_EDGE_RISING>;
+ };
+ };
diff --git a/MAINTAINERS b/MAINTAINERS
index 1157873ccc99..b82e6b670629 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1099,6 +1099,7 @@ L: linux-amlogic@lists.infradead.org
S: Supported
W: http://www.amlogic.com
F: Documentation/admin-guide/perf/meson-ddr-pmu.rst
+F: Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml
F: drivers/perf/amlogic/
F: include/soc/amlogic/
From 7299fdc1cfff0e45e4ca4efd77f250965598b41c Mon Sep 17 00:00:00 2001
From: Jiucheng Xu <jiucheng.xu@amlogic.com>
Date: Tue, 22 Nov 2022 16:40:28 +0800
Subject: [PATCH 081/151] perf/amlogic: Fix build error for x86_64 allmodconfig
The driver misses including <linux/io.h>, which causes a compilation
error with x86_64 'allmodconfig':
drivers/perf/amlogic/meson_g12_ddr_pmu.c: In function 'dmc_g12_get_freq_quick':
drivers/perf/amlogic/meson_g12_ddr_pmu.c:135:15: error: implicit declaration of function 'readl' [-Werror=implicit-function-declaration]
135 | val = readl(info->pll_reg);
| ^~~~~
drivers/perf/amlogic/meson_g12_ddr_pmu.c: In function 'dmc_g12_counter_enable':
drivers/perf/amlogic/meson_g12_ddr_pmu.c:204:9: error: implicit declaration of function 'writel' [-Werror=implicit-function-declaration]
204 | writel(clock_count, info->ddr_reg[0] + DMC_MON_G12_TIMER);
| ^~~~~~
Add the missing header to fix the build.
Fixes: 2016e2113d35 ("perf/amlogic: Add support for Amlogic meson G12 SoC DDR PMU driver")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Jiucheng Xu <jiucheng.xu@amlogic.com>
Reviewed-by: Neil Armstrong <neil.armstrong@linaro.org>
Link: https://lore.kernel.org/r/20221122084028.572494-1-jiucheng.xu@amlogic.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/amlogic/meson_g12_ddr_pmu.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/perf/amlogic/meson_g12_ddr_pmu.c b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
index c07c34f03cce..932802abd18c 100644
--- a/drivers/perf/amlogic/meson_g12_ddr_pmu.c
+++ b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
@@ -4,6 +4,7 @@
*/
#include <linux/err.h>
+#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
From cfce092dae95ed81391e49f273353a96cc6dec64 Mon Sep 17 00:00:00 2001
From: Mark Rutland <mark.rutland@arm.com>
Date: Tue, 22 Nov 2022 16:36:24 +0000
Subject: [PATCH 082/151] ftrace: arm64: remove static ftrace
The build test robot pointer out that there's a build failure when:
CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y
CONFIG_DYNAMIC_FTRACE_WITH_ARGS=n
... due to some mismatched ifdeffery, some of which checks
CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS, and some of which checks
CONFIG_DYNAMIC_FTRACE_WITH_ARGS, leading to some missing definitions expected
by the core code when CONFIG_DYNAMIC_FTRACE=n and consequently
CONFIG_DYNAMIC_FTRACE_WITH_ARGS=n.
There's really not much point in supporting CONFIG_DYNAMIC_FTRACE=n (AKA
static ftrace). All supported toolchains allow us to implement
DYNAMIC_FTRACE, distributions all prefer DYNAMIC_FTRACE, and both
powerpc and s390 removed support for static ftrace in commits:
0c0c52306f4792a4 ("powerpc: Only support DYNAMIC_FTRACE not static")
5d6a0163494c78ad ("s390/ftrace: enforce DYNAMIC_FTRACE if FUNCTION_TRACER is selected")
... and according to Steven, static ftrace is only supported on x86 to
allow testing that the core code still functions in this configuration.
Given that, let's simplify matters by removing arm64's support for
static ftrace. This avoids the problem originally reported, and leaves
us with less code to maintain.
Fixes: 26299b3f6ba2 ("ftrace: arm64: move from REGS to ARGS")
Link: https://lore.kernel.org/r/202211212249.livTPi3Y-lkp@intel.com
Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20221122163624.1225912-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/Kconfig | 1 +
arch/arm64/kernel/entry-ftrace.S | 39 --------------------------------
arch/arm64/kernel/ftrace.c | 5 ----
3 files changed, 1 insertion(+), 44 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b6b3305ba701..5553a734123e 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -117,6 +117,7 @@ config ARM64
select CPU_PM if (SUSPEND || CPU_IDLE)
select CRC32
select DCACHE_WORD_ACCESS
+ select DYNAMIC_FTRACE if FUNCTION_TRACER
select DMA_DIRECT_REMAP
select EDAC_SUPPORT
select FRAME_POINTER
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 4d3050549aa6..30cc2a9d1757 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -170,44 +170,6 @@ SYM_CODE_END(ftrace_caller)
add \reg, \reg, #8
.endm
-#ifndef CONFIG_DYNAMIC_FTRACE
-/*
- * void _mcount(unsigned long return_address)
- * @return_address: return address to instrumented function
- *
- * This function makes calls, if enabled, to:
- * - tracer function to probe instrumented function's entry,
- * - ftrace_graph_caller to set up an exit hook
- */
-SYM_FUNC_START(_mcount)
- mcount_enter
-
- ldr_l x2, ftrace_trace_function
- adr x0, ftrace_stub
- cmp x0, x2 // if (ftrace_trace_function
- b.eq skip_ftrace_call // != ftrace_stub) {
-
- mcount_get_pc x0 // function's pc
- mcount_get_lr x1 // function's lr (= parent's pc)
- blr x2 // (*ftrace_trace_function)(pc, lr);
-
-skip_ftrace_call: // }
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
- ldr_l x2, ftrace_graph_return
- cmp x0, x2 // if ((ftrace_graph_return
- b.ne ftrace_graph_caller // != ftrace_stub)
-
- ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry
- adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
- cmp x0, x2
- b.ne ftrace_graph_caller // ftrace_graph_caller();
-#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
- mcount_exit
-SYM_FUNC_END(_mcount)
-EXPORT_SYMBOL(_mcount)
-NOKPROBE(_mcount)
-
-#else /* CONFIG_DYNAMIC_FTRACE */
/*
* _mcount() is used to build the kernel with -pg option, but all the branch
* instructions to _mcount() are replaced to NOP initially at kernel start up,
@@ -247,7 +209,6 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
mcount_exit
SYM_FUNC_END(ftrace_caller)
-#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/*
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index 5cf990d052ba..b30b955a8921 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -60,7 +60,6 @@ int ftrace_regs_query_register_offset(const char *name)
}
#endif
-#ifdef CONFIG_DYNAMIC_FTRACE
/*
* Replace a single instruction, which may be a branch or NOP.
* If @validate == true, a replaced instruction is checked against 'old'.
@@ -268,7 +267,6 @@ void arch_ftrace_update_code(int command)
command |= FTRACE_MAY_SLEEP;
ftrace_modify_all_code(command);
}
-#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/*
@@ -299,8 +297,6 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
}
}
-#ifdef CONFIG_DYNAMIC_FTRACE
-
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
@@ -338,5 +334,4 @@ int ftrace_disable_ftrace_graph_caller(void)
return ftrace_modify_graph_caller(false);
}
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
-#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
From 223d3a0d30b6e9f979f5642e430e1753d3e29f89 Mon Sep 17 00:00:00 2001
From: Ren Zhijie <renzhijie2@huawei.com>
Date: Thu, 24 Nov 2022 02:24:29 +0000
Subject: [PATCH 083/151] arm64: armv8_deprecated: fix unused-function error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
If CONFIG_SWP_EMULATION is not set and
CONFIG_CP15_BARRIER_EMULATION is not set,
aarch64-linux-gnu complained about unused-function :
arch/arm64/kernel/armv8_deprecated.c:67:21: error: ‘aarch32_check_condition’ defined but not used [-Werror=unused-function]
static unsigned int aarch32_check_condition(u32 opcode, u32 psr)
^~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
To fix this warning, modify aarch32_check_condition() with __maybe_unused.
Fixes: 0c5f416219da ("arm64: armv8_deprecated: move aarch32 helper earlier")
Signed-off-by: Ren Zhijie <renzhijie2@huawei.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221124022429.19024-1-renzhijie2@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/armv8_deprecated.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index ed0788cf6bbb..8a9052cf3013 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -64,7 +64,7 @@ struct insn_emulation {
#define ARM_OPCODE_CONDITION_UNCOND 0xf
-static unsigned int aarch32_check_condition(u32 opcode, u32 psr)
+static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
{
u32 cc_bits = opcode >> 28;
From a8bf2fc43fc63c0bd38b45c9a9616d43b683585d Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Wed, 23 Nov 2022 18:02:08 +0000
Subject: [PATCH 084/151] arm64/kpti: Move DAIF masking to C code
We really don't want to take an exception while replacing TTBR1 so we mask
DAIF during the actual update. Currently this is done in the assembly
function idmap_cpu_replace_ttbr1() but it could equally be done in the only
caller of that function, cpu_replace_ttbr1(). This simplifies the assembly
code slightly and means that when working with the code around masking DAIF
flags there is one less piece of assembly code which needs to be considered.
While we're at it add a comment which makes explicit why we are masking
DAIF in this code.
There should be no functional effect.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221123180209.634650-2-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/mmu_context.h | 10 ++++++++++
arch/arm64/mm/proc.S | 4 ----
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index d3f8b5df0c1f..72dbd6400549 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -18,6 +18,7 @@
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
+#include <asm/daifflags.h>
#include <asm/proc-fns.h>
#include <asm-generic/mm_hooks.h>
#include <asm/cputype.h>
@@ -152,6 +153,7 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
typedef void (ttbr_replace_func)(phys_addr_t);
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
ttbr_replace_func *replace_phys;
+ unsigned long daif;
/* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */
phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
@@ -171,7 +173,15 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
__cpu_install_idmap(idmap);
+
+ /*
+ * We really don't want to take *any* exceptions while TTBR1 is
+ * in the process of being replaced so mask everything.
+ */
+ daif = local_daif_save();
replace_phys(ttbr1);
+ local_daif_restore(daif);
+
cpu_uninstall_idmap();
}
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index b9ecbbae1e1a..066fa60b93d2 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -189,16 +189,12 @@ SYM_FUNC_END(cpu_do_resume)
* called by anything else. It can only be executed from a TTBR0 mapping.
*/
SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
- save_and_disable_daif flags=x2
-
__idmap_cpu_set_reserved_ttbr1 x1, x3
offset_ttbr1 x0, x3
msr ttbr1_el1, x0
isb
- restore_daif x2
-
ret
SYM_FUNC_END(idmap_cpu_replace_ttbr1)
.popsection
From d503d01e5016370be8473fc23d800c7ff37ab7f6 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Wed, 23 Nov 2022 18:02:09 +0000
Subject: [PATCH 085/151] arm64/asm: Remove unused assembler DAIF save/restore
macros
There are no longer any users of the assembler macros for saving and
restoring DAIF so remove them to prevent further users being added, there
are C equivalents available.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221123180209.634650-3-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/assembler.h | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 88175551b401..d8d6627be0f6 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -34,11 +34,6 @@
wx\n .req w\n
.endr
- .macro save_and_disable_daif, flags
- mrs \flags, daif
- msr daifset, #0xf
- .endm
-
.macro disable_daif
msr daifset, #0xf
.endm
@@ -47,10 +42,6 @@
msr daifclr, #0xf
.endm
- .macro restore_daif, flags:req
- msr daif, \flags
- .endm
-
/*
* Save/restore interrupts.
*/
From 453dfcee70c5c344ca09396ff5b0baf177eb327e Mon Sep 17 00:00:00 2001
From: Ard Biesheuvel <ardb@kernel.org>
Date: Tue, 22 Nov 2022 18:02:49 +0100
Subject: [PATCH 086/151] arm64: booting: Require placement within 48-bit
addressable memory
Some configurations (i.e., 64k + LVA/LPA) can tolerate a physical
placement of the kernel image outside of the 48-bit addressable region,
but given that the loader has no way of knowing whether or not the image
in question supports LVA/LPA, it currently has no choice but to place it
below the 48-bit mark.
Once we add support for LPA2, which allows 52-bit physical and virtual
addressing when using 4k or 16k pages, but in way that relies on
increasing the number of paging levels, there will be more variety in
the configurations that may or may not support this.
So redefine bit #3 in the Image header as 'must be placed within 48-bit
addressable memory', as this is the current de facto meaning.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221122170249.2453853-1-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
Documentation/arm64/booting.rst | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst
index 8c324ad638de..5a764fabfea8 100644
--- a/Documentation/arm64/booting.rst
+++ b/Documentation/arm64/booting.rst
@@ -121,8 +121,9 @@ Header notes:
to the base of DRAM, since memory below it is not
accessible via the linear mapping
1
- 2MB aligned base may be anywhere in physical
- memory
+ 2MB aligned base such that all image_size bytes
+ counted from the start of the image are within
+ the 48-bit addressable range of physical memory
Bits 4-63 Reserved.
============= ===============================================================
From 642978981ec8a79da00828c696c58b3732b993a6 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Thu, 24 Nov 2022 12:07:22 +0000
Subject: [PATCH 087/151] kselftest/arm64: Set test names prior to starting
children
Since we now flush output immediately on starting children we should ensure
that the child name is set beforehand so that any output that does get
flushed from the newly created child has the name of the child attached.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221124120722.150988-1-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index ccbfcf847d96..69ca4a5f7e6e 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -290,12 +290,12 @@ static void start_fpsimd(struct child_data *child, int cpu, int copy)
{
int ret;
- child_start(child, "./fpsimd-test");
-
ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy);
if (ret == -1)
ksft_exit_fail_msg("asprintf() failed\n");
+ child_start(child, "./fpsimd-test");
+
ksft_print_msg("Started %s\n", child->name);
}
@@ -307,12 +307,12 @@ static void start_sve(struct child_data *child, int vl, int cpu)
if (ret < 0)
ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl);
- child_start(child, "./sve-test");
-
ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu);
if (ret == -1)
ksft_exit_fail_msg("asprintf() failed\n");
+ child_start(child, "./sve-test");
+
ksft_print_msg("Started %s\n", child->name);
}
@@ -320,16 +320,16 @@ static void start_ssve(struct child_data *child, int vl, int cpu)
{
int ret;
+ ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
+ if (ret == -1)
+ ksft_exit_fail_msg("asprintf() failed\n");
+
ret = prctl(PR_SME_SET_VL, vl | PR_SME_VL_INHERIT);
if (ret < 0)
ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
child_start(child, "./ssve-test");
- ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
- if (ret == -1)
- ksft_exit_fail_msg("asprintf() failed\n");
-
ksft_print_msg("Started %s\n", child->name);
}
@@ -341,12 +341,12 @@ static void start_za(struct child_data *child, int vl, int cpu)
if (ret < 0)
ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
- child_start(child, "./za-test");
-
ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu);
if (ret == -1)
ksft_exit_fail_msg("asprintf() failed\n");
+ child_start(child, "./za-test");
+
ksft_print_msg("Started %s\n", child->name);
}
From ce00d127a6068a0e4e9485424c1fd32564bce326 Mon Sep 17 00:00:00 2001
From: Jiapeng Chong <jiapeng.chong@linux.alibaba.com>
Date: Tue, 29 Nov 2022 11:21:08 +0800
Subject: [PATCH 088/151] perf/amlogic: Remove unused header inclusions of
<linux/version.h>
According to the "Abaci Robot":
| ./drivers/perf/amlogic/meson_g12_ddr_pmu.c:15 linux/version.h not needed.
| ./drivers/perf/amlogic/meson_ddr_pmu_core.c: 19 linux/version.h not needed.
So drop the unnecessary #include directives.
Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=3280
Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=3282
Reported-by: Abaci Robot <abaci@linux.alibaba.com>
Signed-off-by: Jiapeng Chong <jiapeng.chong@linux.alibaba.com>
Link: https://lore.kernel.org/r/20221129032108.119661-1-jiapeng.chong@linux.alibaba.com
Link: https://lore.kernel.org/r/20221129032108.119661-2-jiapeng.chong@linux.alibaba.com
[will: Squashed patches together, filled out commit message a bit more]
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/amlogic/meson_ddr_pmu_core.c | 1 -
drivers/perf/amlogic/meson_g12_ddr_pmu.c | 1 -
2 files changed, 2 deletions(-)
diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c
index 0ff7c0449ac2..b84346dbac2c 100644
--- a/drivers/perf/amlogic/meson_ddr_pmu_core.c
+++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c
@@ -16,7 +16,6 @@
#include <linux/printk.h>
#include <linux/sysfs.h>
#include <linux/types.h>
-#include <linux/version.h>
#include <soc/amlogic/meson_ddr_pmu.h>
diff --git a/drivers/perf/amlogic/meson_g12_ddr_pmu.c b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
index 932802abd18c..a78fdb15e26c 100644
--- a/drivers/perf/amlogic/meson_g12_ddr_pmu.c
+++ b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
@@ -12,7 +12,6 @@
#include <linux/platform_device.h>
#include <linux/printk.h>
#include <linux/types.h>
-#include <linux/version.h>
#include <soc/amlogic/meson_ddr_pmu.h>
From cc91b9481605b1f62f947857231050c747ceda16 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Mon, 28 Nov 2022 08:24:49 +0530
Subject: [PATCH 089/151] arm64/perf: Replace PMU version number '0' with
ID_AA64DFR0_EL1_PMUVer_NI
__armv8pmu_probe_pmu() returns if detected PMU is either not implemented or
implementation defined. Extracted ID_AA64DFR0_EL1_PMUVer value, when PMU is
not implemented is '0' which can be replaced with ID_AA64DFR0_EL1_PMUVer_NI
defined as '0b0000'.
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: linux-perf-users@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20221128025449.39085-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/perf_event.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 7b0643fe2f13..a15b3c1d15d9 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -1146,7 +1146,8 @@ static void __armv8pmu_probe_pmu(void *info)
dfr0 = read_sysreg(id_aa64dfr0_el1);
pmuver = cpuid_feature_extract_unsigned_field(dfr0,
ID_AA64DFR0_EL1_PMUVer_SHIFT);
- if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF || pmuver == 0)
+ if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF ||
+ pmuver == ID_AA64DFR0_EL1_PMUVer_NI)
return;
cpu_pmu->pmuver = pmuver;
From 6b4bb4f38dbfe85247f006f06135ba46450d5bf0 Mon Sep 17 00:00:00 2001
From: Yicong Yang <yangyicong@hisilicon.com>
Date: Thu, 17 Nov 2022 16:41:33 +0800
Subject: [PATCH 090/151] drivers/perf: hisi: Fix some event id for
hisi-pcie-pmu
Some event id of hisi-pcie-pmu is incorrect, fix them.
Fixes: 8404b0fbc7fb ("drivers/perf: hisi: Add driver for HiSilicon PCIe PMU")
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
Link: https://lore.kernel.org/r/20221117084136.53572-2-yangyicong@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/hisilicon/hisi_pcie_pmu.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c
index 21771708597d..071e63d9a9ac 100644
--- a/drivers/perf/hisilicon/hisi_pcie_pmu.c
+++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c
@@ -693,10 +693,10 @@ static struct attribute *hisi_pcie_pmu_events_attr[] = {
HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210),
HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011),
HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011),
- HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x1005),
- HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x11005),
- HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x2004),
- HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x12004),
+ HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804),
+ HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804),
+ HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405),
+ HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405),
NULL
};
From eb79f12b4c41dd2403a0d16772ee72fcd6416015 Mon Sep 17 00:00:00 2001
From: Yicong Yang <yangyicong@hisilicon.com>
Date: Thu, 17 Nov 2022 16:41:34 +0800
Subject: [PATCH 091/151] docs: perf: Fix PMU instance name of hisi-pcie-pmu
The PMU instance will be called hisi_pcie<sicl>_core<core> rather than
hisi_pcie<sicl>_<core>. Fix this in the documentation.
Fixes: c8602008e247 ("docs: perf: Add description for HiSilicon PCIe PMU driver")
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
Link: https://lore.kernel.org/r/20221117084136.53572-3-yangyicong@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
.../admin-guide/perf/hisi-pcie-pmu.rst | 22 +++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
index 294ebbdb22af..bbe66480ff85 100644
--- a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
+++ b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
@@ -15,10 +15,10 @@ HiSilicon PCIe PMU driver
The PCIe PMU driver registers a perf PMU with the name of its sicl-id and PCIe
Core id.::
- /sys/bus/event_source/hisi_pcie<sicl>_<core>
+ /sys/bus/event_source/hisi_pcie<sicl>_core<core>
PMU driver provides description of available events and filter options in sysfs,
-see /sys/bus/event_source/devices/hisi_pcie<sicl>_<core>.
+see /sys/bus/event_source/devices/hisi_pcie<sicl>_core<core>.
The "format" directory describes all formats of the config (events) and config1
(filter options) fields of the perf_event_attr structure. The "events" directory
@@ -33,13 +33,13 @@ monitored by PMU.
Example usage of perf::
$# perf list
- hisi_pcie0_0/rx_mwr_latency/ [kernel PMU event]
- hisi_pcie0_0/rx_mwr_cnt/ [kernel PMU event]
+ hisi_pcie0_core0/rx_mwr_latency/ [kernel PMU event]
+ hisi_pcie0_core0/rx_mwr_cnt/ [kernel PMU event]
------------------------------------------
- $# perf stat -e hisi_pcie0_0/rx_mwr_latency/
- $# perf stat -e hisi_pcie0_0/rx_mwr_cnt/
- $# perf stat -g -e hisi_pcie0_0/rx_mwr_latency/ -e hisi_pcie0_0/rx_mwr_cnt/
+ $# perf stat -e hisi_pcie0_core0/rx_mwr_latency/
+ $# perf stat -e hisi_pcie0_core0/rx_mwr_cnt/
+ $# perf stat -g -e hisi_pcie0_core0/rx_mwr_latency/ -e hisi_pcie0_core0/rx_mwr_cnt/
The current driver does not support sampling. So "perf record" is unsupported.
Also attach to a task is unsupported for PCIe PMU.
@@ -64,7 +64,7 @@ bit8 is set, port=0x100; if these two Root Ports are both monitored, port=0x101.
Example usage of perf::
- $# perf stat -e hisi_pcie0_0/rx_mwr_latency,port=0x1/ sleep 5
+ $# perf stat -e hisi_pcie0_core0/rx_mwr_latency,port=0x1/ sleep 5
-bdf
@@ -76,7 +76,7 @@ For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0.
Example usage of perf::
- $# perf stat -e hisi_pcie0_0/rx_mrd_flux,bdf=0x3900/ sleep 5
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,bdf=0x3900/ sleep 5
2. Trigger filter
Event statistics start when the first time TLP length is greater/smaller
@@ -90,7 +90,7 @@ means start when TLP length < condition.
Example usage of perf::
- $# perf stat -e hisi_pcie0_0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
3. Threshold filter
Counter counts when TLP length within the specified range. You can set the
@@ -103,4 +103,4 @@ when TLP length < threshold.
Example usage of perf::
- $# perf stat -e hisi_pcie0_0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
From c8dff677e6d4221e00e864ef896b488908248b08 Mon Sep 17 00:00:00 2001
From: Bagas Sanjaya <bagasdotme@gmail.com>
Date: Thu, 17 Nov 2022 16:41:35 +0800
Subject: [PATCH 092/151] Documentation: perf: Indent filter options list of
hisi-pcie-pmu
The "Filter options" list have a rather ugly indentation. Also, the first
paragraph after list name is rendered without separator (as continuation
from the name).
Align the list by indenting the list items and add a blank line
separator for each list name.
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Bagas Sanjaya <bagasdotme@gmail.com>
Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
Link: https://lore.kernel.org/r/20221117084136.53572-4-yangyicong@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
.../admin-guide/perf/hisi-pcie-pmu.rst | 80 ++++++++++---------
1 file changed, 43 insertions(+), 37 deletions(-)
diff --git a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
index bbe66480ff85..645f08f65429 100644
--- a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
+++ b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
@@ -48,59 +48,65 @@ Filter options
--------------
1. Target filter
-PMU could only monitor the performance of traffic downstream target Root Ports
-or downstream target Endpoint. PCIe PMU driver support "port" and "bdf"
-interfaces for users, and these two interfaces aren't supported at the same
-time.
--port
-"port" filter can be used in all PCIe PMU events, target Root Port can be
-selected by configuring the 16-bits-bitmap "port". Multi ports can be selected
-for AP-layer-events, and only one port can be selected for TL/DL-layer-events.
+ PMU could only monitor the performance of traffic downstream target Root
+ Ports or downstream target Endpoint. PCIe PMU driver support "port" and
+ "bdf" interfaces for users, and these two interfaces aren't supported at the
+ same time.
-For example, if target Root Port is 0000:00:00.0 (x8 lanes), bit0 of bitmap
-should be set, port=0x1; if target Root Port is 0000:00:04.0 (x4 lanes),
-bit8 is set, port=0x100; if these two Root Ports are both monitored, port=0x101.
+ - port
-Example usage of perf::
+ "port" filter can be used in all PCIe PMU events, target Root Port can be
+ selected by configuring the 16-bits-bitmap "port". Multi ports can be
+ selected for AP-layer-events, and only one port can be selected for
+ TL/DL-layer-events.
- $# perf stat -e hisi_pcie0_core0/rx_mwr_latency,port=0x1/ sleep 5
+ For example, if target Root Port is 0000:00:00.0 (x8 lanes), bit0 of
+ bitmap should be set, port=0x1; if target Root Port is 0000:00:04.0 (x4
+ lanes), bit8 is set, port=0x100; if these two Root Ports are both
+ monitored, port=0x101.
--bdf
+ Example usage of perf::
-"bdf" filter can only be used in bandwidth events, target Endpoint is selected
-by configuring BDF to "bdf". Counter only counts the bandwidth of message
-requested by target Endpoint.
+ $# perf stat -e hisi_pcie0_core0/rx_mwr_latency,port=0x1/ sleep 5
-For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0.
+ - bdf
-Example usage of perf::
+ "bdf" filter can only be used in bandwidth events, target Endpoint is
+ selected by configuring BDF to "bdf". Counter only counts the bandwidth of
+ message requested by target Endpoint.
- $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,bdf=0x3900/ sleep 5
+ For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0.
+
+ Example usage of perf::
+
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,bdf=0x3900/ sleep 5
2. Trigger filter
-Event statistics start when the first time TLP length is greater/smaller
-than trigger condition. You can set the trigger condition by writing "trig_len",
-and set the trigger mode by writing "trig_mode". This filter can only be used
-in bandwidth events.
-For example, "trig_len=4" means trigger condition is 2^4 DW, "trig_mode=0"
-means statistics start when TLP length > trigger condition, "trig_mode=1"
-means start when TLP length < condition.
+ Event statistics start when the first time TLP length is greater/smaller
+ than trigger condition. You can set the trigger condition by writing
+ "trig_len", and set the trigger mode by writing "trig_mode". This filter can
+ only be used in bandwidth events.
-Example usage of perf::
+ For example, "trig_len=4" means trigger condition is 2^4 DW, "trig_mode=0"
+ means statistics start when TLP length > trigger condition, "trig_mode=1"
+ means start when TLP length < condition.
- $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
+ Example usage of perf::
+
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
3. Threshold filter
-Counter counts when TLP length within the specified range. You can set the
-threshold by writing "thr_len", and set the threshold mode by writing
-"thr_mode". This filter can only be used in bandwidth events.
-For example, "thr_len=4" means threshold is 2^4 DW, "thr_mode=0" means
-counter counts when TLP length >= threshold, and "thr_mode=1" means counts
-when TLP length < threshold.
+ Counter counts when TLP length within the specified range. You can set the
+ threshold by writing "thr_len", and set the threshold mode by writing
+ "thr_mode". This filter can only be used in bandwidth events.
-Example usage of perf::
+ For example, "thr_len=4" means threshold is 2^4 DW, "thr_mode=0" means
+ counter counts when TLP length >= threshold, and "thr_mode=1" means counts
+ when TLP length < threshold.
- $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
+ Example usage of perf::
+
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
From 17d573984d4d5ad73c7cb5edcf2024c585475b0c Mon Sep 17 00:00:00 2001
From: Yicong Yang <yangyicong@hisilicon.com>
Date: Thu, 17 Nov 2022 16:41:36 +0800
Subject: [PATCH 093/151] drivers/perf: hisi: Add TLP filter support
The PMU support to filter the TLP when counting the bandwidth with below
options:
- only count the TLP headers
- only count the TLP payloads
- count both TLP headers and payloads
In the current driver it's default to count the TLP payloads only, which
will have an implicity side effects that on the traffic only have header
only TLPs, we'll get no data.
Make this user configuration through "len_mode" parameter and make it
default to count both TLP headers and payloads when user not specified.
Also update the documentation for it.
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
Link: https://lore.kernel.org/r/20221117084136.53572-5-yangyicong@huawei.com
Signed-off-by: Will Deacon <will@kernel.org>
---
.../admin-guide/perf/hisi-pcie-pmu.rst | 18 ++++++++++++++++++
drivers/perf/hisilicon/hisi_pcie_pmu.c | 14 +++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
index 645f08f65429..7e863662e2d4 100644
--- a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
+++ b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst
@@ -110,3 +110,21 @@ Filter options
Example usage of perf::
$# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
+
+4. TLP Length filter
+
+ When counting bandwidth, the data can be composed of certain parts of TLP
+ packets. You can specify it through "len_mode":
+
+ - 2'b00: Reserved (Do not use this since the behaviour is undefined)
+ - 2'b01: Bandwidth of TLP payloads
+ - 2'b10: Bandwidth of TLP headers
+ - 2'b11: Bandwidth of both TLP payloads and headers
+
+ For example, "len_mode=2" means only counting the bandwidth of TLP headers
+ and "len_mode=3" means the final bandwidth data is composed of both TLP
+ headers and payloads. Default value if not specified is 2'b11.
+
+ Example usage of perf::
+
+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,len_mode=0x1/ sleep 5
diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c
index 071e63d9a9ac..6fee0b6e163b 100644
--- a/drivers/perf/hisilicon/hisi_pcie_pmu.c
+++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c
@@ -47,10 +47,14 @@
#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0)
#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27)
#define HISI_PCIE_THR_M GENMASK_ULL(31, 28)
+#define HISI_PCIE_LEN_M GENMASK_ULL(35, 34)
#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36)
#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53)
#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56)
+/* Default config of TLP length mode, will count both TLP headers and payloads */
+#define HISI_PCIE_LEN_M_DEFAULT 3ULL
+
#define HISI_PCIE_MAX_COUNTERS 8
#define HISI_PCIE_REG_STEP 8
#define HISI_PCIE_THR_MAX_VAL 10
@@ -91,6 +95,7 @@ HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0);
HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4);
HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5);
HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9);
+HISI_PCIE_PMU_FILTER_ATTR(len_mode, config1, 11, 10);
HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0);
HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16);
@@ -215,8 +220,8 @@ static void hisi_pcie_pmu_config_filter(struct perf_event *event)
{
struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
struct hw_perf_event *hwc = &event->hw;
+ u64 port, trig_len, thr_len, len_mode;
u64 reg = HISI_PCIE_INIT_SET;
- u64 port, trig_len, thr_len;
/* Config HISI_PCIE_EVENT_CTRL according to event. */
reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event));
@@ -245,6 +250,12 @@ static void hisi_pcie_pmu_config_filter(struct perf_event *event)
reg |= HISI_PCIE_THR_EN;
}
+ len_mode = hisi_pcie_get_len_mode(event);
+ if (len_mode)
+ reg |= FIELD_PREP(HISI_PCIE_LEN_M, len_mode);
+ else
+ reg |= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT);
+
hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg);
}
@@ -711,6 +722,7 @@ static struct attribute *hisi_pcie_pmu_format_attr[] = {
HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"),
HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"),
HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"),
+ HISI_PCIE_PMU_FORMAT_ATTR(len_mode, "config1:10-11"),
HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"),
HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"),
NULL
From 93ae6b01bafee8fa385aa25ee7ebdb40057f6abe Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:33 +0000
Subject: [PATCH 094/151] KVM: arm64: Discard any SVE state when entering KVM
guests
Since 8383741ab2e773a99 (KVM: arm64: Get rid of host SVE tracking/saving)
KVM has not tracked the host SVE state, relying on the fact that we
currently disable SVE whenever we perform a syscall. This may not be true
in future since performance optimisation may result in us keeping SVE
enabled in order to avoid needing to take access traps to reenable it.
Handle this by clearing TIF_SVE and converting the stored task state to
FPSIMD format when preparing to run the guest. This is done with a new
call fpsimd_kvm_prepare() to keep the direct state manipulation
functions internal to fpsimd.c.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-2-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 1 +
arch/arm64/kernel/fpsimd.c | 23 +++++++++++++++++++++++
arch/arm64/kvm/fpsimd.c | 3 ++-
3 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 6f86b7ab6c28..c07e4abaca3d 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -56,6 +56,7 @@ extern void fpsimd_signal_preserve_current_state(void);
extern void fpsimd_preserve_current_state(void);
extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
+extern void fpsimd_kvm_prepare(void);
extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
void *sve_state, unsigned int sve_vl,
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 23834d96d1e7..549e11645e0f 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1627,6 +1627,29 @@ void fpsimd_signal_preserve_current_state(void)
sve_to_fpsimd(current);
}
+/*
+ * Called by KVM when entering the guest.
+ */
+void fpsimd_kvm_prepare(void)
+{
+ if (!system_supports_sve())
+ return;
+
+ /*
+ * KVM does not save host SVE state since we can only enter
+ * the guest from a syscall so the ABI means that only the
+ * non-saved SVE state needs to be saved. If we have left
+ * SVE enabled for performance reasons then update the task
+ * state to be FPSIMD only.
+ */
+ get_cpu_fpsimd_context();
+
+ if (test_and_clear_thread_flag(TIF_SVE))
+ sve_to_fpsimd(current);
+
+ put_cpu_fpsimd_context();
+}
+
/*
* Associate current's FPSIMD context with this cpu
* The caller must have ownership of the cpu FPSIMD context before calling
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index ec8e4494873d..51ca78b31b95 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -75,11 +75,12 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
{
BUG_ON(!current->mm);
- BUG_ON(test_thread_flag(TIF_SVE));
if (!system_supports_fpsimd())
return;
+ fpsimd_kvm_prepare();
+
vcpu->arch.fp_state = FP_STATE_HOST_OWNED;
vcpu_clear_flag(vcpu, HOST_SVE_ENABLED);
From baa8515281b30861cff3da7db70662d2a25c6440 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:34 +0000
Subject: [PATCH 095/151] arm64/fpsimd: Track the saved FPSIMD state type
separately to TIF_SVE
When we save the state for the floating point registers this can be done
in the form visible through either the FPSIMD V registers or the SVE Z and
P registers. At present we track which format is currently used based on
TIF_SVE and the SME streaming mode state but particularly in the SVE case
this limits our options for optimising things, especially around syscalls.
Introduce a new enum which we place together with saved floating point
state in both thread_struct and the KVM guest state which explicitly
states which format is active and keep it up to date when we change it.
At present we do not use this state except to verify that it has the
expected value when loading the state, future patches will introduce
functional changes.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-3-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 2 +-
arch/arm64/include/asm/kvm_host.h | 12 ++++++-
arch/arm64/include/asm/processor.h | 6 ++++
arch/arm64/kernel/fpsimd.c | 58 ++++++++++++++++++++++--------
arch/arm64/kernel/process.c | 2 ++
arch/arm64/kernel/ptrace.c | 3 ++
arch/arm64/kernel/signal.c | 7 +++-
arch/arm64/kvm/fpsimd.c | 3 +-
8 files changed, 74 insertions(+), 19 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index c07e4abaca3d..341705fcb7bb 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -61,7 +61,7 @@ extern void fpsimd_kvm_prepare(void);
extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
void *sve_state, unsigned int sve_vl,
void *za_state, unsigned int sme_vl,
- u64 *svcr);
+ u64 *svcr, enum fp_type *type);
extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 45e2136322ba..fd34ab155d0b 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -306,8 +306,18 @@ struct vcpu_reset_state {
struct kvm_vcpu_arch {
struct kvm_cpu_context ctxt;
- /* Guest floating point state */
+ /*
+ * Guest floating point state
+ *
+ * The architecture has two main floating point extensions,
+ * the original FPSIMD and SVE. These have overlapping
+ * register views, with the FPSIMD V registers occupying the
+ * low 128 bits of the SVE Z registers. When the core
+ * floating point code saves the register state of a task it
+ * records which view it saved in fp_type.
+ */
void *sve_state;
+ enum fp_type fp_type;
unsigned int sve_max_vl;
u64 svcr;
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 445aa3af3b76..3cce0a4c4e8d 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -122,6 +122,11 @@ enum vec_type {
ARM64_VEC_MAX,
};
+enum fp_type {
+ FP_STATE_FPSIMD,
+ FP_STATE_SVE,
+};
+
struct cpu_context {
unsigned long x19;
unsigned long x20;
@@ -152,6 +157,7 @@ struct thread_struct {
struct user_fpsimd_state fpsimd_state;
} uw;
+ enum fp_type fp_type; /* registers FPSIMD or SVE? */
unsigned int fpsimd_cpu;
void *sve_state; /* SVE registers, if any */
void *za_state; /* ZA register, if any */
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 549e11645e0f..aacd8e356084 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -125,6 +125,7 @@ struct fpsimd_last_state_struct {
u64 *svcr;
unsigned int sve_vl;
unsigned int sme_vl;
+ enum fp_type *fp_type;
};
static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
@@ -330,15 +331,6 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
* The task can execute SVE instructions while in userspace without
* trapping to the kernel.
*
- * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
- * corresponding Zn), P0-P15 and FFR are encoded in
- * task->thread.sve_state, formatted appropriately for vector
- * length task->thread.sve_vl or, if SVCR.SM is set,
- * task->thread.sme_vl.
- *
- * task->thread.sve_state must point to a valid buffer at least
- * sve_state_size(task) bytes in size.
- *
* During any syscall, the kernel may optionally clear TIF_SVE and
* discard the vector state except for the FPSIMD subset.
*
@@ -348,7 +340,15 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
* do_sve_acc() to be called, which does some preparation and then
* sets TIF_SVE.
*
- * When stored, FPSIMD registers V0-V31 are encoded in
+ * During any syscall, the kernel may optionally clear TIF_SVE and
+ * discard the vector state except for the FPSIMD subset.
+ *
+ * The data will be stored in one of two formats:
+ *
+ * * FPSIMD only - FP_STATE_FPSIMD:
+ *
+ * When the FPSIMD only state stored task->thread.fp_type is set to
+ * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
* task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
* logically zero but not stored anywhere; P0-P15 and FFR are not
* stored and have unspecified values from userspace's point of
@@ -358,6 +358,19 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
* task->thread.sve_state does not need to be non-NULL, valid or any
* particular size: it must not be dereferenced.
*
+ * * SVE state - FP_STATE_SVE:
+ *
+ * When the full SVE state is stored task->thread.fp_type is set to
+ * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
+ * corresponding Zn), P0-P15 and FFR are encoded in in
+ * task->thread.sve_state, formatted appropriately for vector
+ * length task->thread.sve_vl or, if SVCR.SM is set,
+ * task->thread.sme_vl. The storage for the vector registers in
+ * task->thread.uw.fpsimd_state should be ignored.
+ *
+ * task->thread.sve_state must point to a valid buffer at least
+ * sve_state_size(task) bytes in size.
+ *
* * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
* irrespective of whether TIF_SVE is clear or set, since these are
* not vector length dependent.
@@ -404,12 +417,15 @@ static void task_fpsimd_load(void)
}
}
- if (restore_sve_regs)
+ if (restore_sve_regs) {
+ WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
sve_load_state(sve_pffr(¤t->thread),
¤t->thread.uw.fpsimd_state.fpsr,
restore_ffr);
- else
+ } else {
+ WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
fpsimd_load_state(¤t->thread.uw.fpsimd_state);
+ }
}
/*
@@ -474,8 +490,10 @@ static void fpsimd_save(void)
sve_save_state((char *)last->sve_state +
sve_ffr_offset(vl),
&last->st->fpsr, save_ffr);
+ *last->fp_type = FP_STATE_SVE;
} else {
fpsimd_save_state(last->st);
+ *last->fp_type = FP_STATE_FPSIMD;
}
}
@@ -848,8 +866,10 @@ int vec_set_vector_length(struct task_struct *task, enum vec_type type,
fpsimd_flush_task_state(task);
if (test_and_clear_tsk_thread_flag(task, TIF_SVE) ||
- thread_sm_enabled(&task->thread))
+ thread_sm_enabled(&task->thread)) {
sve_to_fpsimd(task);
+ task->thread.fp_type = FP_STATE_FPSIMD;
+ }
if (system_supports_sme() && type == ARM64_VEC_SME) {
task->thread.svcr &= ~(SVCR_SM_MASK |
@@ -1368,6 +1388,7 @@ static void sve_init_regs(void)
fpsimd_bind_task_to_cpu();
} else {
fpsimd_to_sve(current);
+ current->thread.fp_type = FP_STATE_SVE;
}
}
@@ -1596,6 +1617,8 @@ void fpsimd_flush_thread(void)
current->thread.svcr = 0;
}
+ current->thread.fp_type = FP_STATE_FPSIMD;
+
put_cpu_fpsimd_context();
kfree(sve_state);
kfree(za_state);
@@ -1644,8 +1667,10 @@ void fpsimd_kvm_prepare(void)
*/
get_cpu_fpsimd_context();
- if (test_and_clear_thread_flag(TIF_SVE))
+ if (test_and_clear_thread_flag(TIF_SVE)) {
sve_to_fpsimd(current);
+ current->thread.fp_type = FP_STATE_FPSIMD;
+ }
put_cpu_fpsimd_context();
}
@@ -1667,6 +1692,7 @@ static void fpsimd_bind_task_to_cpu(void)
last->sve_vl = task_get_sve_vl(current);
last->sme_vl = task_get_sme_vl(current);
last->svcr = ¤t->thread.svcr;
+ last->fp_type = ¤t->thread.fp_type;
current->thread.fpsimd_cpu = smp_processor_id();
/*
@@ -1690,7 +1716,8 @@ static void fpsimd_bind_task_to_cpu(void)
void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
unsigned int sve_vl, void *za_state,
- unsigned int sme_vl, u64 *svcr)
+ unsigned int sme_vl, u64 *svcr,
+ enum fp_type *type)
{
struct fpsimd_last_state_struct *last =
this_cpu_ptr(&fpsimd_last_state);
@@ -1704,6 +1731,7 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
last->za_state = za_state;
last->sve_vl = sve_vl;
last->sme_vl = sme_vl;
+ last->fp_type = type;
}
/*
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 044a7d7f1f6a..19cd05eea3f0 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -331,6 +331,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
clear_tsk_thread_flag(dst, TIF_SME);
}
+ dst->thread.fp_type = FP_STATE_FPSIMD;
+
/* clear any pending asynchronous tag fault raised by the parent */
clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index c2fb5755bbec..8a7c91791c16 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -909,6 +909,7 @@ static int sve_set_common(struct task_struct *target,
clear_tsk_thread_flag(target, TIF_SVE);
if (type == ARM64_VEC_SME)
fpsimd_force_sync_to_sve(target);
+ target->thread.fp_type = FP_STATE_FPSIMD;
goto out;
}
@@ -931,6 +932,7 @@ static int sve_set_common(struct task_struct *target,
if (!target->thread.sve_state) {
ret = -ENOMEM;
clear_tsk_thread_flag(target, TIF_SVE);
+ target->thread.fp_type = FP_STATE_FPSIMD;
goto out;
}
@@ -942,6 +944,7 @@ static int sve_set_common(struct task_struct *target,
*/
fpsimd_sync_to_sve(target);
set_tsk_thread_flag(target, TIF_SVE);
+ target->thread.fp_type = FP_STATE_SVE;
BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
start = SVE_PT_SVE_OFFSET;
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 9ad911f1647c..e0d09bf5b01b 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -207,6 +207,7 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
__get_user_error(fpsimd.fpcr, &ctx->fpcr, err);
clear_thread_flag(TIF_SVE);
+ current->thread.fp_type = FP_STATE_FPSIMD;
/* load the hardware registers from the fpsimd_state structure */
if (!err)
@@ -292,6 +293,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user)
if (sve.head.size <= sizeof(*user->sve)) {
clear_thread_flag(TIF_SVE);
current->thread.svcr &= ~SVCR_SM_MASK;
+ current->thread.fp_type = FP_STATE_FPSIMD;
goto fpsimd_only;
}
@@ -327,6 +329,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user)
current->thread.svcr |= SVCR_SM_MASK;
else
set_thread_flag(TIF_SVE);
+ current->thread.fp_type = FP_STATE_SVE;
fpsimd_only:
/* copy the FP and status/control registers */
@@ -932,9 +935,11 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
* FPSIMD register state - flush the saved FPSIMD
* register state in case it gets loaded.
*/
- if (current->thread.svcr & SVCR_SM_MASK)
+ if (current->thread.svcr & SVCR_SM_MASK) {
memset(¤t->thread.uw.fpsimd_state, 0,
sizeof(current->thread.uw.fpsimd_state));
+ current->thread.fp_type = FP_STATE_FPSIMD;
+ }
current->thread.svcr &= ~(SVCR_ZA_MASK |
SVCR_SM_MASK);
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index 51ca78b31b95..a4b4502ad850 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -140,7 +140,8 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
vcpu->arch.sve_state,
vcpu->arch.sve_max_vl,
- NULL, 0, &vcpu->arch.svcr);
+ NULL, 0, &vcpu->arch.svcr,
+ &vcpu->arch.fp_type);
clear_thread_flag(TIF_FOREIGN_FPSTATE);
update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
From deeb8f9a80fdae5a62525656d65c7070c28bd3a4 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:35 +0000
Subject: [PATCH 096/151] arm64/fpsimd: Have KVM explicitly say which FP
registers to save
In order to avoid needlessly saving and restoring the guest registers KVM
relies on the host FPSMID code to save the guest registers when we context
switch away from the guest. This is done by binding the KVM guest state to
the CPU on top of the task state that was originally there, then carefully
managing the TIF_SVE flag for the task to cause the host to save the full
SVE state when needed regardless of the needs of the host task. This works
well enough but isn't terribly direct about what is going on and makes it
much more complicated to try to optimise what we're doing with the SVE
register state.
Let's instead have KVM pass in the register state it wants saving when it
binds to the CPU. We introduce a new FP_STATE_CURRENT for use
during normal task binding to indicate that we should base our
decisions on the current task. This should not be used when
actually saving. Ideally we might want to use a separate enum for
the type to save but this enum and the enum values would then
need to be named which has problems with clarity and ambiguity.
In order to ease any future debugging that might be required this patch
does not actually update any of the decision making about what to save,
it merely starts tracking the new information and warns if the requested
state is not what we would otherwise have decided to save.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-4-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 3 ++-
arch/arm64/include/asm/processor.h | 1 +
arch/arm64/kernel/fpsimd.c | 27 ++++++++++++++++++++++++---
arch/arm64/kvm/fpsimd.c | 9 ++++++++-
4 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 341705fcb7bb..0ad683dab1e2 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -61,7 +61,8 @@ extern void fpsimd_kvm_prepare(void);
extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
void *sve_state, unsigned int sve_vl,
void *za_state, unsigned int sme_vl,
- u64 *svcr, enum fp_type *type);
+ u64 *svcr, enum fp_type *type,
+ enum fp_type to_save);
extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 3cce0a4c4e8d..09f39a2bab47 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -123,6 +123,7 @@ enum vec_type {
};
enum fp_type {
+ FP_STATE_CURRENT, /* Save based on current task state. */
FP_STATE_FPSIMD,
FP_STATE_SVE,
};
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index aacd8e356084..21775f3db58a 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -126,6 +126,7 @@ struct fpsimd_last_state_struct {
unsigned int sve_vl;
unsigned int sme_vl;
enum fp_type *fp_type;
+ enum fp_type to_save;
};
static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
@@ -356,7 +357,8 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
* but userspace is discouraged from relying on this.
*
* task->thread.sve_state does not need to be non-NULL, valid or any
- * particular size: it must not be dereferenced.
+ * particular size: it must not be dereferenced and any data stored
+ * there should be considered stale and not referenced.
*
* * SVE state - FP_STATE_SVE:
*
@@ -369,7 +371,9 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
* task->thread.uw.fpsimd_state should be ignored.
*
* task->thread.sve_state must point to a valid buffer at least
- * sve_state_size(task) bytes in size.
+ * sve_state_size(task) bytes in size. The data stored in
+ * task->thread.uw.fpsimd_state.vregs should be considered stale
+ * and not referenced.
*
* * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
* irrespective of whether TIF_SVE is clear or set, since these are
@@ -459,6 +463,21 @@ static void fpsimd_save(void)
vl = last->sve_vl;
}
+ /*
+ * Validate that an explicitly specified state to save is
+ * consistent with the task state.
+ */
+ switch (last->to_save) {
+ case FP_STATE_CURRENT:
+ break;
+ case FP_STATE_FPSIMD:
+ WARN_ON_ONCE(save_sve_regs);
+ break;
+ case FP_STATE_SVE:
+ WARN_ON_ONCE(!save_sve_regs);
+ break;
+ }
+
if (system_supports_sme()) {
u64 *svcr = last->svcr;
@@ -1693,6 +1712,7 @@ static void fpsimd_bind_task_to_cpu(void)
last->sme_vl = task_get_sme_vl(current);
last->svcr = ¤t->thread.svcr;
last->fp_type = ¤t->thread.fp_type;
+ last->to_save = FP_STATE_CURRENT;
current->thread.fpsimd_cpu = smp_processor_id();
/*
@@ -1717,7 +1737,7 @@ static void fpsimd_bind_task_to_cpu(void)
void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
unsigned int sve_vl, void *za_state,
unsigned int sme_vl, u64 *svcr,
- enum fp_type *type)
+ enum fp_type *type, enum fp_type to_save)
{
struct fpsimd_last_state_struct *last =
this_cpu_ptr(&fpsimd_last_state);
@@ -1732,6 +1752,7 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
last->sve_vl = sve_vl;
last->sme_vl = sme_vl;
last->fp_type = type;
+ last->to_save = to_save;
}
/*
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index a4b4502ad850..89c02ce797b8 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -130,9 +130,16 @@ void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
*/
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
{
+ enum fp_type fp_type;
+
WARN_ON_ONCE(!irqs_disabled());
if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {
+ if (vcpu_has_sve(vcpu))
+ fp_type = FP_STATE_SVE;
+ else
+ fp_type = FP_STATE_FPSIMD;
+
/*
* Currently we do not support SME guests so SVCR is
* always 0 and we just need a variable to point to.
@@ -141,7 +148,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
vcpu->arch.sve_state,
vcpu->arch.sve_max_vl,
NULL, 0, &vcpu->arch.svcr,
- &vcpu->arch.fp_type);
+ &vcpu->arch.fp_type, fp_type);
clear_thread_flag(TIF_FOREIGN_FPSTATE);
update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
From 62021cc36add7b2c015b837f7893f2fb4b8c2586 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:36 +0000
Subject: [PATCH 097/151] arm64/fpsimd: Stop using TIF_SVE to manage register
saving in KVM
Now that we are explicitly telling the host FP code which register state
it needs to save we can remove the manipulation of TIF_SVE from the KVM
code, simplifying it and allowing us to optimise our handling of normal
tasks. Remove the manipulation of TIF_SVE from KVM and instead rely on
to_save to ensure we save the correct data for it.
There should be no functional or performance impact from this change.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-5-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 22 ++++------------------
arch/arm64/kvm/fpsimd.c | 3 ---
2 files changed, 4 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 21775f3db58a..3041e55b1581 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -439,8 +439,8 @@ static void task_fpsimd_load(void)
* last, if KVM is involved this may be the guest VM context rather
* than the host thread for the VM pointed to by current. This means
* that we must always reference the state storage via last rather
- * than via current, other than the TIF_ flags which KVM will
- * carefully maintain for us.
+ * than via current, if we are saving KVM state then it will have
+ * ensured that the type of registers to save is set in last->to_save.
*/
static void fpsimd_save(void)
{
@@ -457,27 +457,13 @@ static void fpsimd_save(void)
if (test_thread_flag(TIF_FOREIGN_FPSTATE))
return;
- if (test_thread_flag(TIF_SVE)) {
+ if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) ||
+ last->to_save == FP_STATE_SVE) {
save_sve_regs = true;
save_ffr = true;
vl = last->sve_vl;
}
- /*
- * Validate that an explicitly specified state to save is
- * consistent with the task state.
- */
- switch (last->to_save) {
- case FP_STATE_CURRENT:
- break;
- case FP_STATE_FPSIMD:
- WARN_ON_ONCE(save_sve_regs);
- break;
- case FP_STATE_SVE:
- WARN_ON_ONCE(!save_sve_regs);
- break;
- }
-
if (system_supports_sme()) {
u64 *svcr = last->svcr;
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index 89c02ce797b8..ec82d0191f76 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -151,7 +151,6 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
&vcpu->arch.fp_type, fp_type);
clear_thread_flag(TIF_FOREIGN_FPSTATE);
- update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
}
}
@@ -208,7 +207,5 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
}
- update_thread_flag(TIF_SVE, 0);
-
local_irq_restore(flags);
}
From a0136be443d51803da4900b52223302f3913812f Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:37 +0000
Subject: [PATCH 098/151] arm64/fpsimd: Load FP state based on recorded data
type
Now that we are recording the type of floating point register state we
are saving when we write the register state out to memory we can use
that information when we load from memory to decide which format to
load, bringing TIF_SVE into line with what we saved rather than relying
on TIF_SVE to determine what to load.
The SME state details are already recorded directly in the saved
SVCR and handled based on the information there.
Since we are not changing any of the save paths there should be no
functional change from this patch, further patches will make use of this
to optimise and clarify the code.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-6-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 40 ++++++++++++++++++++++++++++++--------
1 file changed, 32 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 3041e55b1581..eb42b00dce54 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -395,11 +395,37 @@ static void task_fpsimd_load(void)
WARN_ON(!system_supports_fpsimd());
WARN_ON(!have_cpu_fpsimd_context());
- /* Check if we should restore SVE first */
- if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) {
- sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
- restore_sve_regs = true;
- restore_ffr = true;
+ if (system_supports_sve()) {
+ switch (current->thread.fp_type) {
+ case FP_STATE_FPSIMD:
+ /* Stop tracking SVE for this task until next use. */
+ if (test_and_clear_thread_flag(TIF_SVE))
+ sve_user_disable();
+ break;
+ case FP_STATE_SVE:
+ if (!thread_sm_enabled(¤t->thread) &&
+ !WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)))
+ sve_user_enable();
+
+ if (test_thread_flag(TIF_SVE))
+ sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
+
+ restore_sve_regs = true;
+ restore_ffr = true;
+ break;
+ default:
+ /*
+ * This indicates either a bug in
+ * fpsimd_save() or memory corruption, we
+ * should always record an explicit format
+ * when we save. We always at least have the
+ * memory allocated for FPSMID registers so
+ * try that and hope for the best.
+ */
+ WARN_ON_ONCE(1);
+ clear_thread_flag(TIF_SVE);
+ break;
+ }
}
/* Restore SME, override SVE register configuration if needed */
@@ -415,10 +441,8 @@ static void task_fpsimd_load(void)
if (thread_za_enabled(¤t->thread))
za_load_state(current->thread.za_state);
- if (thread_sm_enabled(¤t->thread)) {
- restore_sve_regs = true;
+ if (thread_sm_enabled(¤t->thread))
restore_ffr = system_supports_fa64();
- }
}
if (restore_sve_regs) {
From bbc6172eefdb276be140415fd6ac6cc8a14a5263 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:38 +0000
Subject: [PATCH 099/151] arm64/fpsimd: SME no longer requires SVE register
state
Now that we track the type of the stored register state separately to
what is active in the task, it is valid to have the FPSIMD register
state stored while in streaming mode. Remove the special case handling
for SME when setting FPSIMD register state.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-7-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 3 +--
arch/arm64/kernel/ptrace.c | 2 --
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index eb42b00dce54..7cb2d89ead83 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -815,8 +815,7 @@ void fpsimd_sync_to_sve(struct task_struct *task)
*/
void sve_sync_to_fpsimd(struct task_struct *task)
{
- if (test_tsk_thread_flag(task, TIF_SVE) ||
- thread_sm_enabled(&task->thread))
+ if (task->thread.fp_type == FP_STATE_SVE)
sve_to_fpsimd(task);
}
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 8a7c91791c16..979dbdc36d52 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -907,8 +907,6 @@ static int sve_set_common(struct task_struct *target,
ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
SVE_PT_FPSIMD_OFFSET);
clear_tsk_thread_flag(target, TIF_SVE);
- if (type == ARM64_VEC_SME)
- fpsimd_force_sync_to_sve(target);
target->thread.fp_type = FP_STATE_FPSIMD;
goto out;
}
From 8c845e2731041f0fdf9287dea80b039b26332c9f Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:39 +0000
Subject: [PATCH 100/151] arm64/sve: Leave SVE enabled on syscall if we don't
context switch
The syscall ABI says that the SVE register state not shared with FPSIMD
may not be preserved on syscall, and this is the only mechanism we have
in the ABI to stop tracking the extra SVE state for a process. Currently
we do this unconditionally by means of disabling SVE for the process on
syscall, causing userspace to take a trap to EL1 if it uses SVE again.
These extra traps result in a noticeable overhead for using SVE instead
of FPSIMD in some workloads, especially for simple syscalls where we can
return directly to userspace and would not otherwise need to update the
floating point registers. Tests with fp-pidbench show an approximately
70% overhead on a range of implementations when SVE is in use - while
this is an extreme and entirely artificial benchmark it is clear that
there is some useful room for improvement here.
Now that we have the ability to track the decision about what to save
seprately to TIF_SVE we can improve things by leaving TIF_SVE enabled on
syscall but only saving the FPSIMD registers if we are in a syscall.
This means that if we need to restore the register state from memory
(eg, after a context switch or kernel mode NEON) we will drop TIF_SVE
and reenable traps for userspace but if we can just return to userspace
then traps will remain disabled.
Since our current implementation and hence ABI has the effect of zeroing
all the SVE register state not shared with FPSIMD on syscall we replace
the disabling of TIF_SVE with a flush of the non-shared register state,
this means that there is still some overhead for syscalls when SVE is in
use but it is very much reduced.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-8-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/fpsimd.c | 8 +++++++-
arch/arm64/kernel/syscall.c | 19 +++++--------------
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 7cb2d89ead83..a66b8640a0a4 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -481,7 +481,13 @@ static void fpsimd_save(void)
if (test_thread_flag(TIF_FOREIGN_FPSTATE))
return;
- if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) ||
+ /*
+ * If a task is in a syscall the ABI allows us to only
+ * preserve the state shared with FPSIMD so don't bother
+ * saving the full SVE state in that case.
+ */
+ if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE) &&
+ !in_syscall(current_pt_regs())) ||
last->to_save == FP_STATE_SVE) {
save_sve_regs = true;
save_ffr = true;
diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c
index d72e8f23422d..a5de47e3df2b 100644
--- a/arch/arm64/kernel/syscall.c
+++ b/arch/arm64/kernel/syscall.c
@@ -183,21 +183,12 @@ static inline void fp_user_discard(void)
if (!system_supports_sve())
return;
- /*
- * If SME is not active then disable SVE, the registers will
- * be cleared when userspace next attempts to access them and
- * we do not need to track the SVE register state until then.
- */
- clear_thread_flag(TIF_SVE);
+ if (test_thread_flag(TIF_SVE)) {
+ unsigned int sve_vq_minus_one;
- /*
- * task_fpsimd_load() won't be called to update CPACR_EL1 in
- * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
- * happens if a context switch or kernel_neon_begin() or context
- * modification (sigreturn, ptrace) intervenes.
- * So, ensure that CPACR_EL1 is already correct for the fast-path case.
- */
- sve_user_disable();
+ sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
+ sve_flush_live(true, sve_vq_minus_one);
+ }
}
void do_el0_svc(struct pt_regs *regs)
From 1192b93ba3528feaf37dc4b5d8d6bdbb475791ad Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 15 Nov 2022 09:46:40 +0000
Subject: [PATCH 101/151] arm64/fp: Use a struct to pass data to
fpsimd_bind_state_to_cpu()
For reasons that are unclear to this reader fpsimd_bind_state_to_cpu()
populates the struct fpsimd_last_state_struct that it uses to store the
active floating point state for KVM guests by passing an argument for
each member of the structure. As the richness of the architecture increases
this is resulting in a function with a rather large number of arguments
which isn't ideal.
Simplify the interface by using the struct directly as the single argument
for the function, renaming it as we lift the definition into the header.
This could be built on further to reduce the work we do adding storage for
new FP state in various places but for now it just simplifies this one
interface.
Signed-off-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20221115094640.112848-9-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 17 ++++++++++++-----
arch/arm64/kernel/fpsimd.c | 34 ++++++---------------------------
arch/arm64/kvm/fpsimd.c | 24 +++++++++++++----------
3 files changed, 32 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 0ad683dab1e2..e6fa1e2982c8 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -58,11 +58,18 @@ extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
extern void fpsimd_kvm_prepare(void);
-extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
- void *sve_state, unsigned int sve_vl,
- void *za_state, unsigned int sme_vl,
- u64 *svcr, enum fp_type *type,
- enum fp_type to_save);
+struct cpu_fp_state {
+ struct user_fpsimd_state *st;
+ void *sve_state;
+ void *za_state;
+ u64 *svcr;
+ unsigned int sve_vl;
+ unsigned int sme_vl;
+ enum fp_type *fp_type;
+ enum fp_type to_save;
+};
+
+extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index a66b8640a0a4..de97d2be915b 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -118,18 +118,8 @@
* returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
* whatever is in the FPSIMD registers is not saved to memory, but discarded.
*/
-struct fpsimd_last_state_struct {
- struct user_fpsimd_state *st;
- void *sve_state;
- void *za_state;
- u64 *svcr;
- unsigned int sve_vl;
- unsigned int sme_vl;
- enum fp_type *fp_type;
- enum fp_type to_save;
-};
-static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
+static DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
#ifdef CONFIG_ARM64_SVE
@@ -468,7 +458,7 @@ static void task_fpsimd_load(void)
*/
static void fpsimd_save(void)
{
- struct fpsimd_last_state_struct const *last =
+ struct cpu_fp_state const *last =
this_cpu_ptr(&fpsimd_last_state);
/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
bool save_sve_regs = false;
@@ -1716,8 +1706,7 @@ void fpsimd_kvm_prepare(void)
*/
static void fpsimd_bind_task_to_cpu(void)
{
- struct fpsimd_last_state_struct *last =
- this_cpu_ptr(&fpsimd_last_state);
+ struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
WARN_ON(!system_supports_fpsimd());
last->st = ¤t->thread.uw.fpsimd_state;
@@ -1749,25 +1738,14 @@ static void fpsimd_bind_task_to_cpu(void)
}
}
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
- unsigned int sve_vl, void *za_state,
- unsigned int sme_vl, u64 *svcr,
- enum fp_type *type, enum fp_type to_save)
+void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state)
{
- struct fpsimd_last_state_struct *last =
- this_cpu_ptr(&fpsimd_last_state);
+ struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
WARN_ON(!system_supports_fpsimd());
WARN_ON(!in_softirq() && !irqs_disabled());
- last->st = st;
- last->svcr = svcr;
- last->sve_state = sve_state;
- last->za_state = za_state;
- last->sve_vl = sve_vl;
- last->sme_vl = sme_vl;
- last->fp_type = type;
- last->to_save = to_save;
+ *last = *state;
}
/*
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index ec82d0191f76..02dd7e9ebd39 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -130,25 +130,29 @@ void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
*/
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
{
- enum fp_type fp_type;
+ struct cpu_fp_state fp_state;
WARN_ON_ONCE(!irqs_disabled());
if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {
- if (vcpu_has_sve(vcpu))
- fp_type = FP_STATE_SVE;
- else
- fp_type = FP_STATE_FPSIMD;
/*
* Currently we do not support SME guests so SVCR is
* always 0 and we just need a variable to point to.
*/
- fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
- vcpu->arch.sve_state,
- vcpu->arch.sve_max_vl,
- NULL, 0, &vcpu->arch.svcr,
- &vcpu->arch.fp_type, fp_type);
+ fp_state.st = &vcpu->arch.ctxt.fp_regs;
+ fp_state.sve_state = vcpu->arch.sve_state;
+ fp_state.sve_vl = vcpu->arch.sve_max_vl;
+ fp_state.za_state = NULL;
+ fp_state.svcr = &vcpu->arch.svcr;
+ fp_state.fp_type = &vcpu->arch.fp_type;
+
+ if (vcpu_has_sve(vcpu))
+ fp_state.to_save = FP_STATE_SVE;
+ else
+ fp_state.to_save = FP_STATE_FPSIMD;
+
+ fpsimd_bind_state_to_cpu(&fp_state);
clear_thread_flag(TIF_FOREIGN_FPSTATE);
}
From 229d58e31678dece365060c50a39a99a2b1dc729 Mon Sep 17 00:00:00 2001
From: Will Deacon <will@kernel.org>
Date: Wed, 16 Nov 2022 17:03:24 +0000
Subject: [PATCH 102/151] firmware: arm_ffa: Move constants to header file
FF-A function IDs and error codes will be needed in the hypervisor too,
so move to them to the header file where they can be shared. Rename the
version constants with an "FFA_" prefix so that they are less likely
to clash with other code in the tree.
Co-developed-by: Andrew Walbran <qwandor@google.com>
Signed-off-by: Andrew Walbran <qwandor@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>
Reviewed-by: Sudeep Holla <sudeep.holla@arm.com>
Link: https://lore.kernel.org/r/20221116170335.2341003-2-qperret@google.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/firmware/arm_ffa/driver.c | 101 +++---------------------------
include/linux/arm_ffa.h | 83 ++++++++++++++++++++++++
2 files changed, 93 insertions(+), 91 deletions(-)
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index d5e86ef40b89..fa85c64d3ded 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -36,81 +36,6 @@
#include "common.h"
#define FFA_DRIVER_VERSION FFA_VERSION_1_0
-
-#define FFA_SMC(calling_convention, func_num) \
- ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, (calling_convention), \
- ARM_SMCCC_OWNER_STANDARD, (func_num))
-
-#define FFA_SMC_32(func_num) FFA_SMC(ARM_SMCCC_SMC_32, (func_num))
-#define FFA_SMC_64(func_num) FFA_SMC(ARM_SMCCC_SMC_64, (func_num))
-
-#define FFA_ERROR FFA_SMC_32(0x60)
-#define FFA_SUCCESS FFA_SMC_32(0x61)
-#define FFA_INTERRUPT FFA_SMC_32(0x62)
-#define FFA_VERSION FFA_SMC_32(0x63)
-#define FFA_FEATURES FFA_SMC_32(0x64)
-#define FFA_RX_RELEASE FFA_SMC_32(0x65)
-#define FFA_RXTX_MAP FFA_SMC_32(0x66)
-#define FFA_FN64_RXTX_MAP FFA_SMC_64(0x66)
-#define FFA_RXTX_UNMAP FFA_SMC_32(0x67)
-#define FFA_PARTITION_INFO_GET FFA_SMC_32(0x68)
-#define FFA_ID_GET FFA_SMC_32(0x69)
-#define FFA_MSG_POLL FFA_SMC_32(0x6A)
-#define FFA_MSG_WAIT FFA_SMC_32(0x6B)
-#define FFA_YIELD FFA_SMC_32(0x6C)
-#define FFA_RUN FFA_SMC_32(0x6D)
-#define FFA_MSG_SEND FFA_SMC_32(0x6E)
-#define FFA_MSG_SEND_DIRECT_REQ FFA_SMC_32(0x6F)
-#define FFA_FN64_MSG_SEND_DIRECT_REQ FFA_SMC_64(0x6F)
-#define FFA_MSG_SEND_DIRECT_RESP FFA_SMC_32(0x70)
-#define FFA_FN64_MSG_SEND_DIRECT_RESP FFA_SMC_64(0x70)
-#define FFA_MEM_DONATE FFA_SMC_32(0x71)
-#define FFA_FN64_MEM_DONATE FFA_SMC_64(0x71)
-#define FFA_MEM_LEND FFA_SMC_32(0x72)
-#define FFA_FN64_MEM_LEND FFA_SMC_64(0x72)
-#define FFA_MEM_SHARE FFA_SMC_32(0x73)
-#define FFA_FN64_MEM_SHARE FFA_SMC_64(0x73)
-#define FFA_MEM_RETRIEVE_REQ FFA_SMC_32(0x74)
-#define FFA_FN64_MEM_RETRIEVE_REQ FFA_SMC_64(0x74)
-#define FFA_MEM_RETRIEVE_RESP FFA_SMC_32(0x75)
-#define FFA_MEM_RELINQUISH FFA_SMC_32(0x76)
-#define FFA_MEM_RECLAIM FFA_SMC_32(0x77)
-#define FFA_MEM_OP_PAUSE FFA_SMC_32(0x78)
-#define FFA_MEM_OP_RESUME FFA_SMC_32(0x79)
-#define FFA_MEM_FRAG_RX FFA_SMC_32(0x7A)
-#define FFA_MEM_FRAG_TX FFA_SMC_32(0x7B)
-#define FFA_NORMAL_WORLD_RESUME FFA_SMC_32(0x7C)
-
-/*
- * For some calls it is necessary to use SMC64 to pass or return 64-bit values.
- * For such calls FFA_FN_NATIVE(name) will choose the appropriate
- * (native-width) function ID.
- */
-#ifdef CONFIG_64BIT
-#define FFA_FN_NATIVE(name) FFA_FN64_##name
-#else
-#define FFA_FN_NATIVE(name) FFA_##name
-#endif
-
-/* FFA error codes. */
-#define FFA_RET_SUCCESS (0)
-#define FFA_RET_NOT_SUPPORTED (-1)
-#define FFA_RET_INVALID_PARAMETERS (-2)
-#define FFA_RET_NO_MEMORY (-3)
-#define FFA_RET_BUSY (-4)
-#define FFA_RET_INTERRUPTED (-5)
-#define FFA_RET_DENIED (-6)
-#define FFA_RET_RETRY (-7)
-#define FFA_RET_ABORTED (-8)
-
-#define MAJOR_VERSION_MASK GENMASK(30, 16)
-#define MINOR_VERSION_MASK GENMASK(15, 0)
-#define MAJOR_VERSION(x) ((u16)(FIELD_GET(MAJOR_VERSION_MASK, (x))))
-#define MINOR_VERSION(x) ((u16)(FIELD_GET(MINOR_VERSION_MASK, (x))))
-#define PACK_VERSION_INFO(major, minor) \
- (FIELD_PREP(MAJOR_VERSION_MASK, (major)) | \
- FIELD_PREP(MINOR_VERSION_MASK, (minor)))
-#define FFA_VERSION_1_0 PACK_VERSION_INFO(1, 0)
#define FFA_MIN_VERSION FFA_VERSION_1_0
#define SENDER_ID_MASK GENMASK(31, 16)
@@ -120,12 +45,6 @@
#define PACK_TARGET_INFO(s, r) \
(FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r)))
-/*
- * FF-A specification mentions explicitly about '4K pages'. This should
- * not be confused with the kernel PAGE_SIZE, which is the translation
- * granule kernel is configured and may be one among 4K, 16K and 64K.
- */
-#define FFA_PAGE_SIZE SZ_4K
/*
* Keeping RX TX buffer size as 4K for now
* 64K may be preferred to keep it min a page in 64K PAGE_SIZE config
@@ -178,9 +97,9 @@ static struct ffa_drv_info *drv_info;
*/
static u32 ffa_compatible_version_find(u32 version)
{
- u16 major = MAJOR_VERSION(version), minor = MINOR_VERSION(version);
- u16 drv_major = MAJOR_VERSION(FFA_DRIVER_VERSION);
- u16 drv_minor = MINOR_VERSION(FFA_DRIVER_VERSION);
+ u16 major = FFA_MAJOR_VERSION(version), minor = FFA_MINOR_VERSION(version);
+ u16 drv_major = FFA_MAJOR_VERSION(FFA_DRIVER_VERSION);
+ u16 drv_minor = FFA_MINOR_VERSION(FFA_DRIVER_VERSION);
if ((major < drv_major) || (major == drv_major && minor <= drv_minor))
return version;
@@ -204,16 +123,16 @@ static int ffa_version_check(u32 *version)
if (ver.a0 < FFA_MIN_VERSION) {
pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n",
- MAJOR_VERSION(ver.a0), MINOR_VERSION(ver.a0),
- MAJOR_VERSION(FFA_MIN_VERSION),
- MINOR_VERSION(FFA_MIN_VERSION));
+ FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0),
+ FFA_MAJOR_VERSION(FFA_MIN_VERSION),
+ FFA_MINOR_VERSION(FFA_MIN_VERSION));
return -EINVAL;
}
- pr_info("Driver version %d.%d\n", MAJOR_VERSION(FFA_DRIVER_VERSION),
- MINOR_VERSION(FFA_DRIVER_VERSION));
- pr_info("Firmware version %d.%d found\n", MAJOR_VERSION(ver.a0),
- MINOR_VERSION(ver.a0));
+ pr_info("Driver version %d.%d\n", FFA_MAJOR_VERSION(FFA_DRIVER_VERSION),
+ FFA_MINOR_VERSION(FFA_DRIVER_VERSION));
+ pr_info("Firmware version %d.%d found\n", FFA_MAJOR_VERSION(ver.a0),
+ FFA_MINOR_VERSION(ver.a0));
*version = ffa_compatible_version_find(ver.a0);
return 0;
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index 5f02d2e6b9d9..daff44d777fa 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -11,6 +11,89 @@
#include <linux/types.h>
#include <linux/uuid.h>
+#define FFA_SMC(calling_convention, func_num) \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, (calling_convention), \
+ ARM_SMCCC_OWNER_STANDARD, (func_num))
+
+#define FFA_SMC_32(func_num) FFA_SMC(ARM_SMCCC_SMC_32, (func_num))
+#define FFA_SMC_64(func_num) FFA_SMC(ARM_SMCCC_SMC_64, (func_num))
+
+#define FFA_ERROR FFA_SMC_32(0x60)
+#define FFA_SUCCESS FFA_SMC_32(0x61)
+#define FFA_INTERRUPT FFA_SMC_32(0x62)
+#define FFA_VERSION FFA_SMC_32(0x63)
+#define FFA_FEATURES FFA_SMC_32(0x64)
+#define FFA_RX_RELEASE FFA_SMC_32(0x65)
+#define FFA_RXTX_MAP FFA_SMC_32(0x66)
+#define FFA_FN64_RXTX_MAP FFA_SMC_64(0x66)
+#define FFA_RXTX_UNMAP FFA_SMC_32(0x67)
+#define FFA_PARTITION_INFO_GET FFA_SMC_32(0x68)
+#define FFA_ID_GET FFA_SMC_32(0x69)
+#define FFA_MSG_POLL FFA_SMC_32(0x6A)
+#define FFA_MSG_WAIT FFA_SMC_32(0x6B)
+#define FFA_YIELD FFA_SMC_32(0x6C)
+#define FFA_RUN FFA_SMC_32(0x6D)
+#define FFA_MSG_SEND FFA_SMC_32(0x6E)
+#define FFA_MSG_SEND_DIRECT_REQ FFA_SMC_32(0x6F)
+#define FFA_FN64_MSG_SEND_DIRECT_REQ FFA_SMC_64(0x6F)
+#define FFA_MSG_SEND_DIRECT_RESP FFA_SMC_32(0x70)
+#define FFA_FN64_MSG_SEND_DIRECT_RESP FFA_SMC_64(0x70)
+#define FFA_MEM_DONATE FFA_SMC_32(0x71)
+#define FFA_FN64_MEM_DONATE FFA_SMC_64(0x71)
+#define FFA_MEM_LEND FFA_SMC_32(0x72)
+#define FFA_FN64_MEM_LEND FFA_SMC_64(0x72)
+#define FFA_MEM_SHARE FFA_SMC_32(0x73)
+#define FFA_FN64_MEM_SHARE FFA_SMC_64(0x73)
+#define FFA_MEM_RETRIEVE_REQ FFA_SMC_32(0x74)
+#define FFA_FN64_MEM_RETRIEVE_REQ FFA_SMC_64(0x74)
+#define FFA_MEM_RETRIEVE_RESP FFA_SMC_32(0x75)
+#define FFA_MEM_RELINQUISH FFA_SMC_32(0x76)
+#define FFA_MEM_RECLAIM FFA_SMC_32(0x77)
+#define FFA_MEM_OP_PAUSE FFA_SMC_32(0x78)
+#define FFA_MEM_OP_RESUME FFA_SMC_32(0x79)
+#define FFA_MEM_FRAG_RX FFA_SMC_32(0x7A)
+#define FFA_MEM_FRAG_TX FFA_SMC_32(0x7B)
+#define FFA_NORMAL_WORLD_RESUME FFA_SMC_32(0x7C)
+
+/*
+ * For some calls it is necessary to use SMC64 to pass or return 64-bit values.
+ * For such calls FFA_FN_NATIVE(name) will choose the appropriate
+ * (native-width) function ID.
+ */
+#ifdef CONFIG_64BIT
+#define FFA_FN_NATIVE(name) FFA_FN64_##name
+#else
+#define FFA_FN_NATIVE(name) FFA_##name
+#endif
+
+/* FFA error codes. */
+#define FFA_RET_SUCCESS (0)
+#define FFA_RET_NOT_SUPPORTED (-1)
+#define FFA_RET_INVALID_PARAMETERS (-2)
+#define FFA_RET_NO_MEMORY (-3)
+#define FFA_RET_BUSY (-4)
+#define FFA_RET_INTERRUPTED (-5)
+#define FFA_RET_DENIED (-6)
+#define FFA_RET_RETRY (-7)
+#define FFA_RET_ABORTED (-8)
+
+/* FFA version encoding */
+#define FFA_MAJOR_VERSION_MASK GENMASK(30, 16)
+#define FFA_MINOR_VERSION_MASK GENMASK(15, 0)
+#define FFA_MAJOR_VERSION(x) ((u16)(FIELD_GET(FFA_MAJOR_VERSION_MASK, (x))))
+#define FFA_MINOR_VERSION(x) ((u16)(FIELD_GET(FFA_MINOR_VERSION_MASK, (x))))
+#define FFA_PACK_VERSION_INFO(major, minor) \
+ (FIELD_PREP(FFA_MAJOR_VERSION_MASK, (major)) | \
+ FIELD_PREP(FFA_MINOR_VERSION_MASK, (minor)))
+#define FFA_VERSION_1_0 FFA_PACK_VERSION_INFO(1, 0)
+
+/**
+ * FF-A specification mentions explicitly about '4K pages'. This should
+ * not be confused with the kernel PAGE_SIZE, which is the translation
+ * granule kernel is configured and may be one among 4K, 16K and 64K.
+ */
+#define FFA_PAGE_SIZE SZ_4K
+
/* FFA Bus/Device/Driver related */
struct ffa_device {
int vm_id;
From c8e320b00a2a720862b9c028153c681b1a50aa61 Mon Sep 17 00:00:00 2001
From: Will Deacon <will@kernel.org>
Date: Wed, 16 Nov 2022 17:03:25 +0000
Subject: [PATCH 103/151] firmware: arm_ffa: Move comment before the field it
is documenting
This is consistent with the other comments in the struct.
Co-developed-by: Andrew Walbran <qwandor@google.com>
Signed-off-by: Andrew Walbran <qwandor@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>
Reviewed-by: Sudeep Holla <sudeep.holla@arm.com>
Link: https://lore.kernel.org/r/20221116170335.2341003-3-qperret@google.com
Signed-off-by: Will Deacon <will@kernel.org>
---
include/linux/arm_ffa.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index daff44d777fa..c87aeecaa9b2 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -244,11 +244,11 @@ struct ffa_mem_region_attributes {
*/
#define FFA_MEM_RETRIEVE_SELF_BORROWER BIT(0)
u8 flag;
- u32 composite_off;
/*
* Offset in bytes from the start of the outer `ffa_memory_region` to
* an `struct ffa_mem_region_addr_range`.
*/
+ u32 composite_off;
u64 reserved;
};
From 37622bae3db306ee472acb02fa55c48f14ddd5a1 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:00 +0000
Subject: [PATCH 104/151] arm64/sysreg: Standardise naming for ID_MMFR0_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates. The scripts would like to follow exactly what is in the
arm-arm, which uses lower case for some of these feature names.
Ensure symbols for the ID_MMFR0_EL1 register have an _EL1 suffix,
and use lower case in feature names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-2-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 16 ++++++++--------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7d301700d1a9..d757e518b08c 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -733,14 +733,14 @@
#define ID_ISAR6_DP_SHIFT 4
#define ID_ISAR6_JSCVT_SHIFT 0
-#define ID_MMFR0_INNERSHR_SHIFT 28
-#define ID_MMFR0_FCSE_SHIFT 24
-#define ID_MMFR0_AUXREG_SHIFT 20
-#define ID_MMFR0_TCM_SHIFT 16
-#define ID_MMFR0_SHARELVL_SHIFT 12
-#define ID_MMFR0_OUTERSHR_SHIFT 8
-#define ID_MMFR0_PMSA_SHIFT 4
-#define ID_MMFR0_VMSA_SHIFT 0
+#define ID_MMFR0_EL1_InnerShr_SHIFT 28
+#define ID_MMFR0_EL1_FCSE_SHIFT 24
+#define ID_MMFR0_EL1_AuxReg_SHIFT 20
+#define ID_MMFR0_EL1_TCM_SHIFT 16
+#define ID_MMFR0_EL1_ShareLvl_SHIFT 12
+#define ID_MMFR0_EL1_OuterShr_SHIFT 8
+#define ID_MMFR0_EL1_PMSA_SHIFT 4
+#define ID_MMFR0_EL1_VMSA_SHIFT 0
#define ID_MMFR4_EVT_SHIFT 28
#define ID_MMFR4_CCIDX_SHIFT 24
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b3f37e2209ad..e42466c9aa0b 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -402,14 +402,14 @@ struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
};
static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
- S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_INNERSHR_SHIFT, 4, 0xf),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_FCSE_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_AUXREG_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_TCM_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_SHARELVL_SHIFT, 4, 0),
- S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_OUTERSHR_SHIFT, 4, 0xf),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_PMSA_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_VMSA_SHIFT, 4, 0),
+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 5ea1534ec320ffaca144136cca3880877738e6d2 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:01 +0000
Subject: [PATCH 105/151] arm64/sysreg: Standardise naming for ID_MMFR4_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_MMFR4_EL1 register have an _EL1 suffix,
and use lower case in feature names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-3-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 16 ++++++++--------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index d757e518b08c..f7d003b975c5 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -742,14 +742,14 @@
#define ID_MMFR0_EL1_PMSA_SHIFT 4
#define ID_MMFR0_EL1_VMSA_SHIFT 0
-#define ID_MMFR4_EVT_SHIFT 28
-#define ID_MMFR4_CCIDX_SHIFT 24
-#define ID_MMFR4_LSM_SHIFT 20
-#define ID_MMFR4_HPDS_SHIFT 16
-#define ID_MMFR4_CNP_SHIFT 12
-#define ID_MMFR4_XNX_SHIFT 8
-#define ID_MMFR4_AC2_SHIFT 4
-#define ID_MMFR4_SPECSEI_SHIFT 0
+#define ID_MMFR4_EL1_EVT_SHIFT 28
+#define ID_MMFR4_EL1_CCIDX_SHIFT 24
+#define ID_MMFR4_EL1_LSM_SHIFT 20
+#define ID_MMFR4_EL1_HPDS_SHIFT 16
+#define ID_MMFR4_EL1_CnP_SHIFT 12
+#define ID_MMFR4_EL1_XNX_SHIFT 8
+#define ID_MMFR4_EL1_AC2_SHIFT 4
+#define ID_MMFR4_EL1_SpecSEI_SHIFT 0
#define ID_MMFR5_ETS_SHIFT 0
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index e42466c9aa0b..b04500da4379 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -491,13 +491,13 @@ static const struct arm64_ftr_bits ftr_id_isar5[] = {
};
static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EVT_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CCIDX_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_LSM_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_HPDS_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CNP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_XNX_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_AC2_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
/*
* SpecSEI = 1 indicates that the PE might generate an SError on an
@@ -505,7 +505,7 @@ static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
* SError might be generated than it will not be. Hence it has been
* classified as FTR_HIGHER_SAFE.
*/
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_SPECSEI_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 7b24177c631de0d75fd7473ee75666498faf225e Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:02 +0000
Subject: [PATCH 106/151] arm64/sysreg: Standardise naming for ID_MMFR5_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_MMFR5_EL1 register have an _EL1 suffix.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-4-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 2 +-
arch/arm64/kernel/cpufeature.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f7d003b975c5..7a719c9804fc 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -751,7 +751,7 @@
#define ID_MMFR4_EL1_AC2_SHIFT 4
#define ID_MMFR4_EL1_SpecSEI_SHIFT 0
-#define ID_MMFR5_ETS_SHIFT 0
+#define ID_MMFR5_EL1_ETS_SHIFT 0
#define ID_PFR0_DIT_SHIFT 24
#define ID_PFR0_CSV2_SHIFT 16
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b04500da4379..29239ea0f139 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -522,7 +522,7 @@ static const struct arm64_ftr_bits ftr_id_isar4[] = {
};
static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_ETS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 52b3dc559a4c240c33c58affdf8247bab485d422 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:03 +0000
Subject: [PATCH 107/151] arm64/sysreg: Standardise naming for ID_ISAR0_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_ISAR0_EL1 register have an _EL1 suffix,
and use lower-case for feature names where the arm-arm does the same.
To functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-5-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 14 +++++++-------
arch/arm64/kernel/cpufeature.c | 14 +++++++-------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7a719c9804fc..4dc80e00a467 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -710,13 +710,13 @@
#define ID_DFR1_MTPMU_SHIFT 0
-#define ID_ISAR0_DIVIDE_SHIFT 24
-#define ID_ISAR0_DEBUG_SHIFT 20
-#define ID_ISAR0_COPROC_SHIFT 16
-#define ID_ISAR0_CMPBRANCH_SHIFT 12
-#define ID_ISAR0_BITFIELD_SHIFT 8
-#define ID_ISAR0_BITCOUNT_SHIFT 4
-#define ID_ISAR0_SWAP_SHIFT 0
+#define ID_ISAR0_EL1_Divide_SHIFT 24
+#define ID_ISAR0_EL1_Debug_SHIFT 20
+#define ID_ISAR0_EL1_Coproc_SHIFT 16
+#define ID_ISAR0_EL1_CmpBranch_SHIFT 12
+#define ID_ISAR0_EL1_BitField_SHIFT 8
+#define ID_ISAR0_EL1_BitCount_SHIFT 4
+#define ID_ISAR0_EL1_Swap_SHIFT 0
#define ID_ISAR5_RDM_SHIFT 24
#define ID_ISAR5_CRC32_SHIFT 16
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 29239ea0f139..122c6e317660 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -470,13 +470,13 @@ static const struct arm64_ftr_bits ftr_gmid[] = {
};
static const struct arm64_ftr_bits ftr_id_isar0[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_COPROC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_CMPBRANCH_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITFIELD_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITCOUNT_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_SWAP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 3f08e378f00e1b1c14763fe225bf553b4733db67 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:04 +0000
Subject: [PATCH 108/151] arm64/sysreg: Standardise naming for ID_ISAR4_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_ISAR4_EL1 register have an _EL1 suffix,
and use lower-case for feature names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-6-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 18 +++++++++---------
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 4dc80e00a467..259756bad5fe 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -699,14 +699,14 @@
#define ID_DFR0_PERFMON_8_4 0x5
#define ID_DFR0_PERFMON_8_5 0x6
-#define ID_ISAR4_SWP_FRAC_SHIFT 28
-#define ID_ISAR4_PSR_M_SHIFT 24
-#define ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT 20
-#define ID_ISAR4_BARRIER_SHIFT 16
-#define ID_ISAR4_SMC_SHIFT 12
-#define ID_ISAR4_WRITEBACK_SHIFT 8
-#define ID_ISAR4_WITHSHIFTS_SHIFT 4
-#define ID_ISAR4_UNPRIV_SHIFT 0
+#define ID_ISAR4_EL1_SWP_frac_SHIFT 28
+#define ID_ISAR4_EL1_PSR_M_SHIFT 24
+#define ID_ISAR4_EL1_SynchPrim_frac_SHIFT 20
+#define ID_ISAR4_EL1_Barrier_SHIFT 16
+#define ID_ISAR4_EL1_SMC_SHIFT 12
+#define ID_ISAR4_EL1_Writeback_SHIFT 8
+#define ID_ISAR4_EL1_WithShifts_SHIFT 4
+#define ID_ISAR4_EL1_Unpriv_SHIFT 0
#define ID_DFR1_MTPMU_SHIFT 0
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 122c6e317660..9d0424aed30a 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -510,14 +510,14 @@ static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
};
static const struct arm64_ftr_bits ftr_id_isar4[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SWP_FRAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_PSR_M_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_BARRIER_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SMC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WRITEBACK_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WITHSHIFTS_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_UNPRIV_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -1119,7 +1119,7 @@ static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
* EL1-dependent register fields to avoid spurious sanity check fails.
*/
if (!id_aa64pfr0_32bit_el1(pfr0)) {
- relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_SMC_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);
relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);
relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);
From 816c8638d8c66991dd6dacd32e578293d10393f4 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:05 +0000
Subject: [PATCH 109/151] arm64/sysreg: Standardise naming for ID_ISAR5_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_ISAR5_EL1 register have an _EL1 suffix.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-7-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 12 ++++++------
arch/arm64/kernel/cpufeature.c | 22 +++++++++++-----------
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 259756bad5fe..dbd376174223 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -718,12 +718,12 @@
#define ID_ISAR0_EL1_BitCount_SHIFT 4
#define ID_ISAR0_EL1_Swap_SHIFT 0
-#define ID_ISAR5_RDM_SHIFT 24
-#define ID_ISAR5_CRC32_SHIFT 16
-#define ID_ISAR5_SHA2_SHIFT 12
-#define ID_ISAR5_SHA1_SHIFT 8
-#define ID_ISAR5_AES_SHIFT 4
-#define ID_ISAR5_SEVL_SHIFT 0
+#define ID_ISAR5_EL1_RDM_SHIFT 24
+#define ID_ISAR5_EL1_CRC32_SHIFT 16
+#define ID_ISAR5_EL1_SHA2_SHIFT 12
+#define ID_ISAR5_EL1_SHA1_SHIFT 8
+#define ID_ISAR5_EL1_AES_SHIFT 4
+#define ID_ISAR5_EL1_SEVL_SHIFT 0
#define ID_ISAR6_I8MM_SHIFT 24
#define ID_ISAR6_BF16_SHIFT 20
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9d0424aed30a..efa3fcece38c 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -481,12 +481,12 @@ static const struct arm64_ftr_bits ftr_id_isar0[] = {
};
static const struct arm64_ftr_bits ftr_id_isar5[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -2842,11 +2842,11 @@ static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
- HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
- HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
- HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
- HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
- HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
#endif
{},
};
From eef4344f779f6aee9ec9d49a03ec2317a85a744e Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:06 +0000
Subject: [PATCH 110/151] arm64/sysreg: Standardise naming for ID_ISAR6_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_ISAR6_EL1 register have an _EL1 suffix.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-8-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 14 +++++++-------
arch/arm64/kernel/cpufeature.c | 14 +++++++-------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index dbd376174223..10a00b219851 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -725,13 +725,13 @@
#define ID_ISAR5_EL1_AES_SHIFT 4
#define ID_ISAR5_EL1_SEVL_SHIFT 0
-#define ID_ISAR6_I8MM_SHIFT 24
-#define ID_ISAR6_BF16_SHIFT 20
-#define ID_ISAR6_SPECRES_SHIFT 16
-#define ID_ISAR6_SB_SHIFT 12
-#define ID_ISAR6_FHM_SHIFT 8
-#define ID_ISAR6_DP_SHIFT 4
-#define ID_ISAR6_JSCVT_SHIFT 0
+#define ID_ISAR6_EL1_I8MM_SHIFT 24
+#define ID_ISAR6_EL1_BF16_SHIFT 20
+#define ID_ISAR6_EL1_SPECRES_SHIFT 16
+#define ID_ISAR6_EL1_SB_SHIFT 12
+#define ID_ISAR6_EL1_FHM_SHIFT 8
+#define ID_ISAR6_EL1_DP_SHIFT 4
+#define ID_ISAR6_EL1_JSCVT_SHIFT 0
#define ID_MMFR0_EL1_InnerShr_SHIFT 28
#define ID_MMFR0_EL1_FCSE_SHIFT 24
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index efa3fcece38c..63ecad8f1730 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -527,13 +527,13 @@ static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
};
static const struct arm64_ftr_bits ftr_id_isar6[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_I8MM_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_BF16_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SPECRES_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SB_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_FHM_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_DP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_JSCVT_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
ARM64_FTR_END,
};
From e0bf98fef3fd0f934deee3ebc3a03b88aec5b501 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:07 +0000
Subject: [PATCH 111/151] arm64/sysreg: Standardise naming for ID_PFR0_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_PFR0_EL1 register have an _EL1 suffix,
and use lower case in feature names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-9-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 12 ++++++------
arch/arm64/kernel/cpufeature.c | 12 ++++++------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 10a00b219851..6b4975132db9 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -753,12 +753,12 @@
#define ID_MMFR5_EL1_ETS_SHIFT 0
-#define ID_PFR0_DIT_SHIFT 24
-#define ID_PFR0_CSV2_SHIFT 16
-#define ID_PFR0_STATE3_SHIFT 12
-#define ID_PFR0_STATE2_SHIFT 8
-#define ID_PFR0_STATE1_SHIFT 4
-#define ID_PFR0_STATE0_SHIFT 0
+#define ID_PFR0_EL1_DIT_SHIFT 24
+#define ID_PFR0_EL1_CSV2_SHIFT 16
+#define ID_PFR0_EL1_State3_SHIFT 12
+#define ID_PFR0_EL1_State2_SHIFT 8
+#define ID_PFR0_EL1_State1_SHIFT 4
+#define ID_PFR0_EL1_State0_SHIFT 0
#define ID_DFR0_PERFMON_SHIFT 24
#define ID_DFR0_MPROFDBG_SHIFT 20
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 63ecad8f1730..11e84a4f8b97 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -538,12 +538,12 @@ static const struct arm64_ftr_bits ftr_id_isar6[] = {
};
static const struct arm64_ftr_bits ftr_id_pfr0[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_DIT_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_CSV2_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE3_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE2_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE1_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE0_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 0a648056d68d4049dfb5d1c36b79ad3eba0090f0 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:08 +0000
Subject: [PATCH 112/151] arm64/sysreg: Standardise naming for ID_PFR1_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_PFR1_EL1 register have an _EL1 suffix,
and use lower case in feature names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-10-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 26 +++++++++++++-------------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 6b4975132db9..f93f68ebdccc 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -789,14 +789,14 @@
#define MVFR1_FPDNAN_SHIFT 4
#define MVFR1_FPFTZ_SHIFT 0
-#define ID_PFR1_GIC_SHIFT 28
-#define ID_PFR1_VIRT_FRAC_SHIFT 24
-#define ID_PFR1_SEC_FRAC_SHIFT 20
-#define ID_PFR1_GENTIMER_SHIFT 16
-#define ID_PFR1_VIRTUALIZATION_SHIFT 12
-#define ID_PFR1_MPROGMOD_SHIFT 8
-#define ID_PFR1_SECURITY_SHIFT 4
-#define ID_PFR1_PROGMOD_SHIFT 0
+#define ID_PFR1_EL1_GIC_SHIFT 28
+#define ID_PFR1_EL1_Virt_frac_SHIFT 24
+#define ID_PFR1_EL1_Sec_frac_SHIFT 20
+#define ID_PFR1_EL1_GenTimer_SHIFT 16
+#define ID_PFR1_EL1_Virtualization_SHIFT 12
+#define ID_PFR1_EL1_MProgMod_SHIFT 8
+#define ID_PFR1_EL1_Security_SHIFT 4
+#define ID_PFR1_EL1_ProgMod_SHIFT 0
#if defined(CONFIG_ARM64_4K_PAGES)
#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 11e84a4f8b97..271a82dd59d4 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -548,14 +548,14 @@ static const struct arm64_ftr_bits ftr_id_pfr0[] = {
};
static const struct arm64_ftr_bits ftr_id_pfr1[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GIC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRT_FRAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SEC_FRAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GENTIMER_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRTUALIZATION_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_MPROGMOD_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SECURITY_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_PROGMOD_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -1120,11 +1120,11 @@ static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
*/
if (!id_aa64pfr0_32bit_el1(pfr0)) {
relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
- relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);
- relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);
- relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);
- relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SECURITY_SHIFT);
- relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_PROGMOD_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
}
taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
From 1ecf3dcb136320359ffd79d1ca5908e078443b6b Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:09 +0000
Subject: [PATCH 113/151] arm64/sysreg: Standardise naming for ID_PFR2_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_PFR2_EL1 register have an _EL1 suffix.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-11-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 4 ++--
arch/arm64/kernel/cpufeature.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f93f68ebdccc..155cb298c897 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -768,8 +768,8 @@
#define ID_DFR0_COPSDBG_SHIFT 4
#define ID_DFR0_COPDBG_SHIFT 0
-#define ID_PFR2_SSBS_SHIFT 4
-#define ID_PFR2_CSV3_SHIFT 0
+#define ID_PFR2_EL1_SSBS_SHIFT 4
+#define ID_PFR2_EL1_CSV3_SHIFT 0
#define MVFR0_FPROUND_SHIFT 28
#define MVFR0_FPSHVEC_SHIFT 24
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 271a82dd59d4..8009fc2e4b5e 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -560,8 +560,8 @@ static const struct arm64_ftr_bits ftr_id_pfr1[] = {
};
static const struct arm64_ftr_bits ftr_id_pfr2[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
ARM64_FTR_END,
};
From f4f5969e354235bbe729ff3881089fc902c0daff Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:10 +0000
Subject: [PATCH 114/151] arm64/sysreg: Standardise naming for ID_DFR0_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_DFR0_EL1 register have an _EL1 suffix,
and use lower-case for feature names where the arm-arm does the same.
The arm-arm has feature names for some of the ID_DFR0_EL1.PerMon encodings.
Use these feature names in preference to the '8_4' indication of the
architecture version they were introduced in.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-12-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 24 +++++++++++-------------
arch/arm64/kernel/cpufeature.c | 14 +++++++-------
arch/arm64/kvm/sys_regs.c | 4 ++--
3 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 155cb298c897..835b279f7f20 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -692,12 +692,10 @@
#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
#endif
-#define ID_DFR0_PERFMON_SHIFT 24
-
-#define ID_DFR0_PERFMON_8_0 0x3
-#define ID_DFR0_PERFMON_8_1 0x4
-#define ID_DFR0_PERFMON_8_4 0x5
-#define ID_DFR0_PERFMON_8_5 0x6
+#define ID_DFR0_EL1_PerfMon_PMUv3 0x3
+#define ID_DFR0_EL1_PerfMon_PMUv3p1 0x4
+#define ID_DFR0_EL1_PerfMon_PMUv3p4 0x5
+#define ID_DFR0_EL1_PerfMon_PMUv3p5 0x6
#define ID_ISAR4_EL1_SWP_frac_SHIFT 28
#define ID_ISAR4_EL1_PSR_M_SHIFT 24
@@ -760,13 +758,13 @@
#define ID_PFR0_EL1_State1_SHIFT 4
#define ID_PFR0_EL1_State0_SHIFT 0
-#define ID_DFR0_PERFMON_SHIFT 24
-#define ID_DFR0_MPROFDBG_SHIFT 20
-#define ID_DFR0_MMAPTRC_SHIFT 16
-#define ID_DFR0_COPTRC_SHIFT 12
-#define ID_DFR0_MMAPDBG_SHIFT 8
-#define ID_DFR0_COPSDBG_SHIFT 4
-#define ID_DFR0_COPDBG_SHIFT 0
+#define ID_DFR0_EL1_PerfMon_SHIFT 24
+#define ID_DFR0_EL1_MProfDbg_SHIFT 20
+#define ID_DFR0_EL1_MMapTrc_SHIFT 16
+#define ID_DFR0_EL1_CopTrc_SHIFT 12
+#define ID_DFR0_EL1_MMapDbg_SHIFT 8
+#define ID_DFR0_EL1_CopSDbg_SHIFT 4
+#define ID_DFR0_EL1_CopDbg_SHIFT 0
#define ID_PFR2_EL1_SSBS_SHIFT 4
#define ID_PFR2_EL1_CSV3_SHIFT 0
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 8009fc2e4b5e..77b65a75ba07 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -567,13 +567,13 @@ static const struct arm64_ftr_bits ftr_id_pfr2[] = {
static const struct arm64_ftr_bits ftr_id_dfr0[] = {
/* [31:28] TraceFilt */
- S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_PERFMON_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MPROFDBG_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPTRC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPTRC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPDBG_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPSDBG_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPDBG_SHIFT, 4, 0),
+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
ARM64_FTR_END,
};
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index f4a7c5abcbca..608e4f25161d 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1121,8 +1121,8 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r
case SYS_ID_DFR0_EL1:
/* Limit guests to PMUv3 for ARMv8.4 */
val = cpuid_feature_cap_perfmon_field(val,
- ID_DFR0_PERFMON_SHIFT,
- kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
+ ID_DFR0_EL1_PerfMon_SHIFT,
+ kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_EL1_PerfMon_PMUv3p4 : 0);
break;
}
From d092106d73539905b5deadfa86be04ff141bcaa8 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:11 +0000
Subject: [PATCH 115/151] arm64/sysreg: Standardise naming for ID_DFR1_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the ID_DFR1_EL1 register have an _EL1 suffix.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-13-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 2 +-
arch/arm64/kernel/cpufeature.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 835b279f7f20..29d93a36eac9 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -706,7 +706,7 @@
#define ID_ISAR4_EL1_WithShifts_SHIFT 4
#define ID_ISAR4_EL1_Unpriv_SHIFT 0
-#define ID_DFR1_MTPMU_SHIFT 0
+#define ID_DFR1_EL1_MTPMU_SHIFT 0
#define ID_ISAR0_EL1_Divide_SHIFT 24
#define ID_ISAR0_EL1_Debug_SHIFT 20
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 77b65a75ba07..84b3dc994a70 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -578,7 +578,7 @@ static const struct arm64_ftr_bits ftr_id_dfr0[] = {
};
static const struct arm64_ftr_bits ftr_id_dfr1[] = {
- S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_MTPMU_SHIFT, 4, 0),
+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
ARM64_FTR_END,
};
From a3aab94801dee86cc9d7d43a611c71688a483908 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:12 +0000
Subject: [PATCH 116/151] arm64/sysreg: Standardise naming for MVFR0_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the MVFR0_EL1 register use lower-case for feature
names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-14-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 20 ++++++++++----------
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 29d93a36eac9..0d4ab1c78d22 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -769,14 +769,14 @@
#define ID_PFR2_EL1_SSBS_SHIFT 4
#define ID_PFR2_EL1_CSV3_SHIFT 0
-#define MVFR0_FPROUND_SHIFT 28
-#define MVFR0_FPSHVEC_SHIFT 24
-#define MVFR0_FPSQRT_SHIFT 20
-#define MVFR0_FPDIVIDE_SHIFT 16
-#define MVFR0_FPTRAP_SHIFT 12
-#define MVFR0_FPDP_SHIFT 8
-#define MVFR0_FPSP_SHIFT 4
-#define MVFR0_SIMD_SHIFT 0
+#define MVFR0_EL1_FPRound_SHIFT 28
+#define MVFR0_EL1_FPShVec_SHIFT 24
+#define MVFR0_EL1_FPSqrt_SHIFT 20
+#define MVFR0_EL1_FPDivide_SHIFT 16
+#define MVFR0_EL1_FPTrap_SHIFT 12
+#define MVFR0_EL1_FPDP_SHIFT 8
+#define MVFR0_EL1_FPSP_SHIFT 4
+#define MVFR0_EL1_SIMDReg_SHIFT 0
#define MVFR1_SIMDFMAC_SHIFT 28
#define MVFR1_FPHP_SHIFT 24
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 84b3dc994a70..7ffe076ce7a6 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -429,14 +429,14 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
};
static const struct arm64_ftr_bits ftr_mvfr0[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPROUND_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSHVEC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSQRT_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDIVIDE_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPTRAP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_SIMD_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -2840,8 +2840,8 @@ static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
- HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
- HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
From d3e1aa85b1b27b0e6d6f8d8b0bf0a9a04ed1d40a Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:13 +0000
Subject: [PATCH 117/151] arm64/sysreg: Standardise naming for MVFR1_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the MVFR1_EL1 register use lower-case for feature
names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-15-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 16 ++++++++--------
arch/arm64/kernel/cpufeature.c | 24 ++++++++++++------------
2 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 0d4ab1c78d22..5f2dfbc4a347 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -778,14 +778,14 @@
#define MVFR0_EL1_FPSP_SHIFT 4
#define MVFR0_EL1_SIMDReg_SHIFT 0
-#define MVFR1_SIMDFMAC_SHIFT 28
-#define MVFR1_FPHP_SHIFT 24
-#define MVFR1_SIMDHP_SHIFT 20
-#define MVFR1_SIMDSP_SHIFT 16
-#define MVFR1_SIMDINT_SHIFT 12
-#define MVFR1_SIMDLS_SHIFT 8
-#define MVFR1_FPDNAN_SHIFT 4
-#define MVFR1_FPFTZ_SHIFT 0
+#define MVFR1_EL1_SIMDFMAC_SHIFT 28
+#define MVFR1_EL1_FPHP_SHIFT 24
+#define MVFR1_EL1_SIMDHP_SHIFT 20
+#define MVFR1_EL1_SIMDSP_SHIFT 16
+#define MVFR1_EL1_SIMDInt_SHIFT 12
+#define MVFR1_EL1_SIMDLS_SHIFT 8
+#define MVFR1_EL1_FPDNaN_SHIFT 4
+#define MVFR1_EL1_FPFtZ_SHIFT 0
#define ID_PFR1_EL1_GIC_SHIFT 28
#define ID_PFR1_EL1_Virt_frac_SHIFT 24
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 7ffe076ce7a6..c56339066304 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -441,14 +441,14 @@ static const struct arm64_ftr_bits ftr_mvfr0[] = {
};
static const struct arm64_ftr_bits ftr_mvfr1[] = {
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDFMAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPHP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDHP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDSP_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDINT_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDLS_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPDNAN_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPFTZ_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
ARM64_FTR_END,
};
@@ -2829,16 +2829,16 @@ static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope)
else
mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
- return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDSP_SHIFT) &&
- cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDINT_SHIFT) &&
- cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDLS_SHIFT);
+ return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
}
#endif
static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
#ifdef CONFIG_COMPAT
HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
- HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
+ HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_EL1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
From c6e155e8e561dafcba9faf86598d7ec69942ab86 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:14 +0000
Subject: [PATCH 118/151] arm64/sysreg: Standardise naming for MVFR2_EL1
To convert the 32bit id registers to use the sysreg generation, they
must first have a regular pattern, to match the symbols the script
generates.
Ensure symbols for the MVFR2_EL1 register use lower-case for feature
names where the arm-arm does the same.
No functional change.
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-16-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 4 ++--
arch/arm64/kernel/cpufeature.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 5f2dfbc4a347..fb9f5db0b936 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -813,8 +813,8 @@
#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
#endif
-#define MVFR2_FPMISC_SHIFT 4
-#define MVFR2_SIMDMISC_SHIFT 0
+#define MVFR2_EL1_FPMisc_SHIFT 4
+#define MVFR2_EL1_SIMDMisc_SHIFT 0
#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */
#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index c56339066304..935579e5517a 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -453,8 +453,8 @@ static const struct arm64_ftr_bits ftr_mvfr1[] = {
};
static const struct arm64_ftr_bits ftr_mvfr2[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
ARM64_FTR_END,
};
From 7587cdef55928ef70d4e945003646711ad39e405 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:15 +0000
Subject: [PATCH 119/151] arm64/sysreg: Extend the maximum width of a register
and symbol name
32bit has multiple values for its id registers, as extra properties
were added to the CPUs. Some of these end up having long names, which
exceed the fixed 48 character column that the sysreg awk script generates.
For example, the ID_MMFR1_EL1.L1Hvd field has an encoding whose natural
name would be 'invalidate Iside only'. Using this causes compile errors
as the script generates the following:
#define ID_MMFR1_EL1_L1Hvd_INVALIDATE_ISIDE_ONLYUL(0b0001)
Add a few extra characters.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-17-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/tools/gen-sysreg.awk | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/tools/gen-sysreg.awk b/arch/arm64/tools/gen-sysreg.awk
index db461921d256..c350164a3955 100755
--- a/arch/arm64/tools/gen-sysreg.awk
+++ b/arch/arm64/tools/gen-sysreg.awk
@@ -33,7 +33,7 @@ function expect_fields(nf) {
# Print a CPP macro definition, padded with spaces so that the macro bodies
# line up in a column
function define(name, val) {
- printf "%-48s%s\n", "#define " name, val
+ printf "%-56s%s\n", "#define " name, val
}
# Print standard BITMASK/SHIFT/WIDTH CPP definitions for a field
From 8893df290e3684e1900db52700d77a8ed78e2904 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:16 +0000
Subject: [PATCH 120/151] arm64/sysreg: Convert ID_MMFR0_EL1 to automatic
generation
Convert ID_MMFR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-18-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 -------
arch/arm64/tools/sysreg | 47 +++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index fb9f5db0b936..f856fbf8bfdf 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -171,7 +171,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
@@ -731,15 +730,6 @@
#define ID_ISAR6_EL1_DP_SHIFT 4
#define ID_ISAR6_EL1_JSCVT_SHIFT 0
-#define ID_MMFR0_EL1_InnerShr_SHIFT 28
-#define ID_MMFR0_EL1_FCSE_SHIFT 24
-#define ID_MMFR0_EL1_AuxReg_SHIFT 20
-#define ID_MMFR0_EL1_TCM_SHIFT 16
-#define ID_MMFR0_EL1_ShareLvl_SHIFT 12
-#define ID_MMFR0_EL1_OuterShr_SHIFT 8
-#define ID_MMFR0_EL1_PMSA_SHIFT 4
-#define ID_MMFR0_EL1_VMSA_SHIFT 0
-
#define ID_MMFR4_EL1_EVT_SHIFT 28
#define ID_MMFR4_EL1_CCIDX_SHIFT 24
#define ID_MMFR4_EL1_LSM_SHIFT 20
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 384757a7eda9..5f2273768173 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -46,6 +46,53 @@
# feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
# item ACCDATA) though it may be more taseful to do something else.
+Sysreg ID_MMFR0_EL1 3 0 0 1 4
+Res0 63:32
+Enum 31:28 InnerShr
+ 0b0000 NC
+ 0b0001 HW
+ 0b1111 IGNORED
+EndEnum
+Enum 27:24 FCSE
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 AuxReg
+ 0b0000 NI
+ 0b0001 ACTLR
+ 0b0010 AIFSR
+EndEnum
+Enum 19:16 TCM
+ 0b0000 NI
+ 0b0001 IMPDEF
+ 0b0010 TCM
+ 0b0011 TCM_DMA
+EndEnum
+Enum 15:12 ShareLvl
+ 0b0000 ONE
+ 0b0001 TWO
+EndEnum
+Enum 11:8 OuterShr
+ 0b0000 NC
+ 0b0001 HW
+ 0b1111 IGNORED
+EndEnum
+Enum 7:4 PMSA
+ 0b0000 NI
+ 0b0001 IMPDEF
+ 0b0010 PMSAv6
+ 0b0011 PMSAv7
+EndEnum
+Enum 3:0 VMSA
+ 0b0000 NI
+ 0b0001 IMPDEF
+ 0b0010 VMSAv6
+ 0b0011 VMSAv7
+ 0b0100 VMSAv7_PXN
+ 0b0101 VMSAv7_LONG
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From 7e2f00bea3dbbe082cf2ff33ca8711235d098207 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:17 +0000
Subject: [PATCH 121/151] arm64/sysreg: Convert ID_MMFR1_EL1 to automatic
generation
Convert ID_MMFR1_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-19-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 49 +++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f856fbf8bfdf..e7e8400fc61f 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -171,7 +171,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 5f2273768173..2f1d0077afed 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -93,6 +93,55 @@ Enum 3:0 VMSA
EndEnum
EndSysreg
+Sysreg ID_MMFR1_EL1 3 0 0 1 5
+Res0 63:32
+Enum 31:28 BPred
+ 0b0000 NI
+ 0b0001 BP_SW_MANGED
+ 0b0010 BP_ASID_AWARE
+ 0b0011 BP_NOSNOOP
+ 0b0100 BP_INVISIBLE
+EndEnum
+Enum 27:24 L1TstCln
+ 0b0000 NI
+ 0b0001 NOINVALIDATE
+ 0b0010 INVALIDATE
+EndEnum
+Enum 23:20 L1Uni
+ 0b0000 NI
+ 0b0001 INVALIDATE
+ 0b0010 CLEAN_AND_INVALIDATE
+EndEnum
+Enum 19:16 L1Hvd
+ 0b0000 NI
+ 0b0001 INVALIDATE_ISIDE_ONLY
+ 0b0010 INVALIDATE
+ 0b0011 CLEAN_AND_INVALIDATE
+EndEnum
+Enum 15:12 L1UniSW
+ 0b0000 NI
+ 0b0001 CLEAN
+ 0b0010 CLEAN_AND_INVALIDATE
+ 0b0011 INVALIDATE
+EndEnum
+Enum 11:8 L1HvdSW
+ 0b0000 NI
+ 0b0001 CLEAN_AND_INVALIDATE
+ 0b0010 INVALIDATE_DSIDE_ONLY
+ 0b0011 INVALIDATE
+EndEnum
+Enum 7:4 L1UniVA
+ 0b0000 NI
+ 0b0001 CLEAN_AND_INVALIDATE
+ 0b0010 INVALIDATE_BP
+EndEnum
+Enum 3:0 L1HvdVA
+ 0b0000 NI
+ 0b0001 CLEAN_AND_INVALIDATE
+ 0b0010 INVALIDATE_BP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From fbfba88b6ae136a6ae02dddd36bcf8f66f811128 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:18 +0000
Subject: [PATCH 122/151] arm64/sysreg: Convert ID_MMFR2_EL1 to automatic
generation
Convert ID_MMFR2_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-20-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 41 +++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index e7e8400fc61f..7e7e647a895c 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -171,7 +171,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 2f1d0077afed..df8b234eb7e9 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -142,6 +142,47 @@ Enum 3:0 L1HvdVA
EndEnum
EndSysreg
+Sysreg ID_MMFR2_EL1 3 0 0 1 6
+Res0 63:32
+Enum 31:28 HWAccFlg
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 WFIStall
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 MemBarr
+ 0b0000 NI
+ 0b0001 DSB_ONLY
+ 0b0010 IMP
+EndEnum
+Enum 19:16 UniTLB
+ 0b0000 NI
+ 0b0001 BY_VA
+ 0b0010 BY_MATCH_ASID
+ 0b0011 BY_ALL_ASID
+ 0b0100 OTHER_TLBS
+ 0b0101 BROADCAST
+ 0b0110 BY_IPA
+EndEnum
+Enum 15:12 HvdTLB
+ 0b0000 NI
+EndEnum
+Enum 11:8 L1HvdRng
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 L1HvdBG
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 L1HvdFG
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From 8fe2a9c578b07c9a631fbf7cd9e278ffa6619869 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:19 +0000
Subject: [PATCH 123/151] arm64/sysreg: Convert ID_MMFR3_EL1 to automatic
generation
Convert ID_MMFR3_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-21-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 40 +++++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7e7e647a895c..be4015d22a30 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -171,7 +171,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index df8b234eb7e9..7573312ef249 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -183,6 +183,46 @@ Enum 3:0 L1HvdFG
EndEnum
EndSysreg
+Sysreg ID_MMFR3_EL1 3 0 0 1 7
+Res0 63:32
+Enum 31:28 Supersec
+ 0b0000 IMP
+ 0b1111 NI
+EndEnum
+Enum 27:24 CMemSz
+ 0b0000 4GB
+ 0b0001 64GB
+ 0b0010 1TB
+EndEnum
+Enum 23:20 CohWalk
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 PAN
+ 0b0000 NI
+ 0b0001 PAN
+ 0b0010 PAN2
+EndEnum
+Enum 15:12 MaintBcst
+ 0b0000 NI
+ 0b0001 NO_TLB
+ 0b0010 ALL
+EndEnum
+Enum 11:8 BPMaint
+ 0b0000 NI
+ 0b0001 ALL
+ 0b0010 BY_VA
+EndEnum
+Enum 7:4 CMaintSW
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 CMaintVA
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From 5b380ae0e2b3f354bd1274008286b10ee8585015 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:20 +0000
Subject: [PATCH 124/151] arm64/sysreg: Convert ID_MMFR4_EL1 to automatic
generation
Convert ID_MMFR4_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-22-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ---------
arch/arm64/tools/sysreg | 38 +++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index be4015d22a30..b9fdb1e0a451 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -171,7 +171,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
@@ -727,15 +726,6 @@
#define ID_ISAR6_EL1_DP_SHIFT 4
#define ID_ISAR6_EL1_JSCVT_SHIFT 0
-#define ID_MMFR4_EL1_EVT_SHIFT 28
-#define ID_MMFR4_EL1_CCIDX_SHIFT 24
-#define ID_MMFR4_EL1_LSM_SHIFT 20
-#define ID_MMFR4_EL1_HPDS_SHIFT 16
-#define ID_MMFR4_EL1_CnP_SHIFT 12
-#define ID_MMFR4_EL1_XNX_SHIFT 8
-#define ID_MMFR4_EL1_AC2_SHIFT 4
-#define ID_MMFR4_EL1_SpecSEI_SHIFT 0
-
#define ID_MMFR5_EL1_ETS_SHIFT 0
#define ID_PFR0_EL1_DIT_SHIFT 24
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7573312ef249..7cd258ee7a0f 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -223,6 +223,44 @@ Enum 3:0 CMaintVA
EndEnum
EndSysreg
+Sysreg ID_MMFR4_EL1 3 0 0 2 6
+Res0 63:32
+Enum 31:28 EVT
+ 0b0000 NI
+ 0b0001 NO_TLBIS
+ 0b0010 TLBIS
+EndEnum
+Enum 27:24 CCIDX
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 LSM
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 HPDS
+ 0b0000 NI
+ 0b0001 AA32HPD
+ 0b0010 HPDS2
+EndEnum
+Enum 15:12 CnP
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 XNX
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 AC2
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 SpecSEI
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From 258a96b25a9d2ee54171653b801637edc3cc0b7f Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:21 +0000
Subject: [PATCH 125/151] arm64/sysreg: Convert ID_ISAR0_EL1 to automatic
generation
Convert ID_ISAR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-23-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 9 ---------
arch/arm64/tools/sysreg | 36 +++++++++++++++++++++++++++++++++
2 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index b9fdb1e0a451..758f1c139c25 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
@@ -703,14 +702,6 @@
#define ID_DFR1_EL1_MTPMU_SHIFT 0
-#define ID_ISAR0_EL1_Divide_SHIFT 24
-#define ID_ISAR0_EL1_Debug_SHIFT 20
-#define ID_ISAR0_EL1_Coproc_SHIFT 16
-#define ID_ISAR0_EL1_CmpBranch_SHIFT 12
-#define ID_ISAR0_EL1_BitField_SHIFT 8
-#define ID_ISAR0_EL1_BitCount_SHIFT 4
-#define ID_ISAR0_EL1_Swap_SHIFT 0
-
#define ID_ISAR5_EL1_RDM_SHIFT 24
#define ID_ISAR5_EL1_CRC32_SHIFT 16
#define ID_ISAR5_EL1_SHA2_SHIFT 12
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7cd258ee7a0f..bc1133af1746 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -223,6 +223,42 @@ Enum 3:0 CMaintVA
EndEnum
EndSysreg
+Sysreg ID_ISAR0_EL1 3 0 0 2 0
+Res0 63:28
+Enum 27:24 Divide
+ 0b0000 NI
+ 0b0001 xDIV_T32
+ 0b0010 xDIV_A32
+EndEnum
+Enum 23:20 Debug
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 Coproc
+ 0b0000 NI
+ 0b0001 MRC
+ 0b0010 MRC2
+ 0b0011 MRRC
+ 0b0100 MRRC2
+EndEnum
+Enum 15:12 CmpBranch
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 BitField
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 BitCount
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 Swap
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From 892386a6a8073aa880ab4b89e0fe421fa429b7bb Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:22 +0000
Subject: [PATCH 126/151] arm64/sysreg: Convert ID_ISAR1_EL1 to automatic
generation
Convert ID_ISAR1_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-24-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 39 +++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 758f1c139c25..d15edee3b1da 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index bc1133af1746..d7c7949be47f 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -259,6 +259,45 @@ Enum 3:0 Swap
EndEnum
EndSysreg
+Sysreg ID_ISAR1_EL1 3 0 0 2 1
+Res0 63:32
+Enum 31:28 Jazelle
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 Interwork
+ 0b0000 NI
+ 0b0001 BX
+ 0b0010 BLX
+ 0b0011 A32_BX
+EndEnum
+Enum 23:20 Immediate
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 IfThen
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 Extend
+ 0b0000 NI
+ 0b0001 SXTB
+ 0b0010 SXTB16
+EndEnum
+Enum 11:8 Except_AR
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 Except
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 Endian
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From dfa70ae8d8c2b24ecb55b8877f693f9b0a13fb95 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:23 +0000
Subject: [PATCH 127/151] arm64/sysreg: Convert ID_ISAR2_EL1 to automatic
generation
Convert ID_ISAR2_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-25-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 46 +++++++++++++++++++++++++++++++++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index d15edee3b1da..f62d21a00da4 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index d7c7949be47f..9a7e5d0e63eb 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -298,6 +298,52 @@ Enum 3:0 Endian
EndEnum
EndSysreg
+Sysreg ID_ISAR2_EL1 3 0 0 2 2
+Res0 63:32
+Enum 31:28 Reversal
+ 0b0000 NI
+ 0b0001 REV
+ 0b0010 RBIT
+EndEnum
+Enum 27:24 PSR_AR
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 MultU
+ 0b0000 NI
+ 0b0001 UMULL
+ 0b0010 UMAAL
+EndEnum
+Enum 19:16 MultS
+ 0b0000 NI
+ 0b0001 SMULL
+ 0b0010 SMLABB
+ 0b0011 SMLAD
+EndEnum
+Enum 15:12 Mult
+ 0b0000 NI
+ 0b0001 MLA
+ 0b0010 MLS
+EndEnum
+Enum 11:8 MultiAccessInt
+ 0b0000 NI
+ 0b0001 RESTARTABLE
+ 0b0010 CONTINUABLE
+EndEnum
+Enum 7:4 MemHint
+ 0b0000 NI
+ 0b0001 PLD
+ 0b0010 PLD2
+ 0b0011 PLI
+ 0b0100 PLDW
+EndEnum
+Enum 3:0 LoadStore
+ 0b0000 NI
+ 0b0001 DOUBLE
+ 0b0010 ACQUIRE
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From d07016c965300d5872a751b4f45a8934f576c9ca Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:24 +0000
Subject: [PATCH 128/151] arm64/sysreg: Convert ID_ISAR3_EL1 to automatic
generation
Convert ID_ISAR3_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-26-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 38 +++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f62d21a00da4..bfd69abf1bf7 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 9a7e5d0e63eb..1e2d5a3b619c 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -344,6 +344,44 @@ Enum 3:0 LoadStore
EndEnum
EndSysreg
+Sysreg ID_ISAR3_EL1 3 0 0 2 3
+Res0 63:32
+Enum 31:28 T32EE
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 TrueNOP
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 T32Copy
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 TabBranch
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 SynchPrim
+ 0b0000 NI
+ 0b0001 EXCLUSIVE
+ 0b0010 DOUBLE
+EndEnum
+Enum 11:8 SVC
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 SIMD
+ 0b0000 NI
+ 0b0001 SSAT
+ 0b0011 PKHBT
+EndEnum
+Enum 3:0 Saturate
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From 849cc9bd9f0ef532c208d1cc01a824dc119646e3 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:25 +0000
Subject: [PATCH 129/151] arm64/sysreg: Convert ID_ISAR4_EL1 to automatic
generation
Convert ID_ISAR4_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-27-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ---------
arch/arm64/tools/sysreg | 39 +++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index bfd69abf1bf7..0a63c39407f2 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
@@ -688,15 +687,6 @@
#define ID_DFR0_EL1_PerfMon_PMUv3p4 0x5
#define ID_DFR0_EL1_PerfMon_PMUv3p5 0x6
-#define ID_ISAR4_EL1_SWP_frac_SHIFT 28
-#define ID_ISAR4_EL1_PSR_M_SHIFT 24
-#define ID_ISAR4_EL1_SynchPrim_frac_SHIFT 20
-#define ID_ISAR4_EL1_Barrier_SHIFT 16
-#define ID_ISAR4_EL1_SMC_SHIFT 12
-#define ID_ISAR4_EL1_Writeback_SHIFT 8
-#define ID_ISAR4_EL1_WithShifts_SHIFT 4
-#define ID_ISAR4_EL1_Unpriv_SHIFT 0
-
#define ID_DFR1_EL1_MTPMU_SHIFT 0
#define ID_ISAR5_EL1_RDM_SHIFT 24
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 1e2d5a3b619c..ed88cc1ae2a8 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -382,6 +382,45 @@ Enum 3:0 Saturate
EndEnum
EndSysreg
+Sysreg ID_ISAR4_EL1 3 0 0 2 4
+Res0 63:32
+Enum 31:28 SWP_frac
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 PSR_M
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 SynchPrim_frac
+ 0b0000 NI
+ 0b0011 IMP
+EndEnum
+Enum 19:16 Barrier
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 SMC
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 Writeback
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 WithShifts
+ 0b0000 NI
+ 0b0001 LSL3
+ 0b0011 LS
+ 0b0100 REG
+EndEnum
+Enum 3:0 Unpriv
+ 0b0000 NI
+ 0b0001 REG_BYTE
+ 0b0010 SIGNED_HALFWORD
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From f4e9ce12dd88d33c25019e2053ade587d7b95969 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:26 +0000
Subject: [PATCH 130/151] arm64/sysreg: Convert ID_ISAR5_EL1 to automatic
generation
Convert ID_ISAR5_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-28-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 8 --------
arch/arm64/tools/sysreg | 34 +++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 0a63c39407f2..04a6e44427a9 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,7 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
@@ -689,13 +688,6 @@
#define ID_DFR1_EL1_MTPMU_SHIFT 0
-#define ID_ISAR5_EL1_RDM_SHIFT 24
-#define ID_ISAR5_EL1_CRC32_SHIFT 16
-#define ID_ISAR5_EL1_SHA2_SHIFT 12
-#define ID_ISAR5_EL1_SHA1_SHIFT 8
-#define ID_ISAR5_EL1_AES_SHIFT 4
-#define ID_ISAR5_EL1_SEVL_SHIFT 0
-
#define ID_ISAR6_EL1_I8MM_SHIFT 24
#define ID_ISAR6_EL1_BF16_SHIFT 20
#define ID_ISAR6_EL1_SPECRES_SHIFT 16
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index ed88cc1ae2a8..43e765a2c68f 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -421,6 +421,40 @@ Enum 3:0 Unpriv
EndEnum
EndSysreg
+Sysreg ID_ISAR5_EL1 3 0 0 2 5
+Res0 63:32
+Enum 31:28 VCMA
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 RDM
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Res0 23:20
+Enum 19:16 CRC32
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 SHA2
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 SHA1
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 AES
+ 0b0000 NI
+ 0b0001 IMP
+ 0b0010 VMULL
+EndEnum
+Enum 3:0 SEVL
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From 5ea58a1b5c7a3830322e6921f9e415968ee2af54 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:27 +0000
Subject: [PATCH 131/151] arm64/sysreg: Convert ID_ISAR6_EL1 to automatic
generation
Convert ID_ISAR6_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-29-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ----------
arch/arm64/tools/sysreg | 32 ++++++++++++++++++++++++++++++++
2 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 04a6e44427a9..03f38890cb2b 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -173,8 +173,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
-
#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
@@ -688,14 +686,6 @@
#define ID_DFR1_EL1_MTPMU_SHIFT 0
-#define ID_ISAR6_EL1_I8MM_SHIFT 24
-#define ID_ISAR6_EL1_BF16_SHIFT 20
-#define ID_ISAR6_EL1_SPECRES_SHIFT 16
-#define ID_ISAR6_EL1_SB_SHIFT 12
-#define ID_ISAR6_EL1_FHM_SHIFT 8
-#define ID_ISAR6_EL1_DP_SHIFT 4
-#define ID_ISAR6_EL1_JSCVT_SHIFT 0
-
#define ID_MMFR5_EL1_ETS_SHIFT 0
#define ID_PFR0_EL1_DIT_SHIFT 24
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 43e765a2c68f..aa6b3f5316f0 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -455,6 +455,38 @@ Enum 3:0 SEVL
EndEnum
EndSysreg
+Sysreg ID_ISAR6_EL1 3 0 0 2 7
+Res0 63:28
+Enum 27:24 I8MM
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 BF16
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 SPECRES
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 SB
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 FHM
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 DP
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 JSCVT
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR4_EL1 3 0 0 2 6
Res0 63:32
Enum 31:28 EVT
From fb0b8d1a24d8707e93b2338bf233df5a904cbdf6 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:28 +0000
Subject: [PATCH 132/151] arm64/sysreg: Convert ID_PFR0_EL1 to automatic
generation
Convert ID_PFR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221130171637.718182-30-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 8 -------
arch/arm64/tools/sysreg | 41 +++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 03f38890cb2b..72b32e179b46 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -165,7 +165,6 @@
#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
-#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
@@ -688,13 +687,6 @@
#define ID_MMFR5_EL1_ETS_SHIFT 0
-#define ID_PFR0_EL1_DIT_SHIFT 24
-#define ID_PFR0_EL1_CSV2_SHIFT 16
-#define ID_PFR0_EL1_State3_SHIFT 12
-#define ID_PFR0_EL1_State2_SHIFT 8
-#define ID_PFR0_EL1_State1_SHIFT 4
-#define ID_PFR0_EL1_State0_SHIFT 0
-
#define ID_DFR0_EL1_PerfMon_SHIFT 24
#define ID_DFR0_EL1_MProfDbg_SHIFT 20
#define ID_DFR0_EL1_MMapTrc_SHIFT 16
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index aa6b3f5316f0..641bd7e3365c 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -46,6 +46,47 @@
# feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
# item ACCDATA) though it may be more taseful to do something else.
+Sysreg ID_PFR0_EL1 3 0 0 1 0
+Res0 63:32
+Enum 31:28 RAS
+ 0b0000 NI
+ 0b0001 RAS
+ 0b0010 RASv1p1
+EndEnum
+Enum 27:24 DIT
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 AMU
+ 0b0000 NI
+ 0b0001 AMUv1
+ 0b0010 AMUv1p1
+EndEnum
+Enum 19:16 CSV2
+ 0b0000 UNDISCLOSED
+ 0b0001 IMP
+ 0b0010 CSV2p1
+EndEnum
+Enum 15:12 State3
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 State2
+ 0b0000 NI
+ 0b0001 NO_CV
+ 0b0010 CV
+EndEnum
+Enum 7:4 State1
+ 0b0000 NI
+ 0b0001 THUMB
+ 0b0010 THUMB2
+EndEnum
+Enum 3:0 State0
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR0_EL1 3 0 0 1 4
Res0 63:32
Enum 31:28 InnerShr
From 1224308075f17dc42dc96043dfc031e221b87c98 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:29 +0000
Subject: [PATCH 133/151] arm64/sysreg: Convert ID_PFR1_EL1 to automatic
generation
Convert ID_PFR1_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-31-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ---------
arch/arm64/tools/sysreg | 40 +++++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 72b32e179b46..85197dd180e0 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -165,7 +165,6 @@
#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
-#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
@@ -716,15 +715,6 @@
#define MVFR1_EL1_FPDNaN_SHIFT 4
#define MVFR1_EL1_FPFtZ_SHIFT 0
-#define ID_PFR1_EL1_GIC_SHIFT 28
-#define ID_PFR1_EL1_Virt_frac_SHIFT 24
-#define ID_PFR1_EL1_Sec_frac_SHIFT 20
-#define ID_PFR1_EL1_GenTimer_SHIFT 16
-#define ID_PFR1_EL1_Virtualization_SHIFT 12
-#define ID_PFR1_EL1_MProgMod_SHIFT 8
-#define ID_PFR1_EL1_Security_SHIFT 4
-#define ID_PFR1_EL1_ProgMod_SHIFT 0
-
#if defined(CONFIG_ARM64_4K_PAGES)
#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 641bd7e3365c..e86193b295e4 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -87,6 +87,46 @@ Enum 3:0 State0
EndEnum
EndSysreg
+Sysreg ID_PFR1_EL1 3 0 0 1 1
+Res0 63:32
+Enum 31:28 GIC
+ 0b0000 NI
+ 0b0001 GICv3
+ 0b0010 GICv4p1
+EndEnum
+Enum 27:24 Virt_frac
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 Sec_frac
+ 0b0000 NI
+ 0b0001 WALK_DISABLE
+ 0b0010 SECURE_MEMORY
+EndEnum
+Enum 19:16 GenTimer
+ 0b0000 NI
+ 0b0001 IMP
+ 0b0010 ECV
+EndEnum
+Enum 15:12 Virtualization
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 MProgMod
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 Security
+ 0b0000 NI
+ 0b0001 EL3
+ 0b0001 NSACR_RFR
+EndEnum
+Enum 3:0 ProgMod
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR0_EL1 3 0 0 1 4
Res0 63:32
Enum 31:28 InnerShr
From 039d372305fff8aa7dc22774c80637d57775eee6 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:30 +0000
Subject: [PATCH 134/151] arm64/sysreg: Convert ID_PFR2_EL1 to automatic
generation
Convert ID_PFR2_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-32-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 4 ----
arch/arm64/tools/sysreg | 16 ++++++++++++++++
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 85197dd180e0..ccb64dc09a4e 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -165,7 +165,6 @@
#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
-#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
@@ -694,9 +693,6 @@
#define ID_DFR0_EL1_CopSDbg_SHIFT 4
#define ID_DFR0_EL1_CopDbg_SHIFT 0
-#define ID_PFR2_EL1_SSBS_SHIFT 4
-#define ID_PFR2_EL1_CSV3_SHIFT 0
-
#define MVFR0_EL1_FPRound_SHIFT 28
#define MVFR0_EL1_FPShVec_SHIFT 24
#define MVFR0_EL1_FPSqrt_SHIFT 20
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index e86193b295e4..667428a89578 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -606,6 +606,22 @@ Enum 3:0 SpecSEI
EndEnum
EndSysreg
+Sysreg ID_PFR2_EL1 3 0 0 3 4
+Res0 63:12
+Enum 11:8 RAS_frac
+ 0b0000 NI
+ 0b0001 RASv1p1
+EndEnum
+Enum 7:4 SSBS
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 CSV3
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From e79c94a2a487515aeb1557b6d3e540ae5f66a67a Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:31 +0000
Subject: [PATCH 135/151] arm64/sysreg: Convert MVFR0_EL1 to automatic
generation
Convert MVFR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-33-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ---------
arch/arm64/tools/sysreg | 39 +++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index ccb64dc09a4e..561968f7b66d 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -170,7 +170,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
@@ -693,15 +692,6 @@
#define ID_DFR0_EL1_CopSDbg_SHIFT 4
#define ID_DFR0_EL1_CopDbg_SHIFT 0
-#define MVFR0_EL1_FPRound_SHIFT 28
-#define MVFR0_EL1_FPShVec_SHIFT 24
-#define MVFR0_EL1_FPSqrt_SHIFT 20
-#define MVFR0_EL1_FPDivide_SHIFT 16
-#define MVFR0_EL1_FPTrap_SHIFT 12
-#define MVFR0_EL1_FPDP_SHIFT 8
-#define MVFR0_EL1_FPSP_SHIFT 4
-#define MVFR0_EL1_SIMDReg_SHIFT 0
-
#define MVFR1_EL1_SIMDFMAC_SHIFT 28
#define MVFR1_EL1_FPHP_SHIFT 24
#define MVFR1_EL1_SIMDHP_SHIFT 20
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 667428a89578..7a56a9a0efdf 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -606,6 +606,45 @@ Enum 3:0 SpecSEI
EndEnum
EndSysreg
+Sysreg MVFR0_EL1 3 0 0 3 0
+Res0 63:32
+Enum 31:28 FPRound
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 FPShVec
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 23:20 FPSqrt
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 FPDivide
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 FPTrap
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 FPDP
+ 0b0000 NI
+ 0b0001 VFPv2
+ 0b0001 VFPv3
+EndEnum
+Enum 7:4 FPSP
+ 0b0000 NI
+ 0b0001 VFPv2
+ 0b0001 VFPv3
+EndEnum
+Enum 3:0 SIMDReg
+ 0b0000 NI
+ 0b0001 IMP_16x64
+ 0b0001 IMP_32x64
+EndEnum
+EndSysreg
+
Sysreg ID_PFR2_EL1 3 0 0 3 4
Res0 63:12
Enum 11:8 RAS_frac
From c9b718eda706179310821daf48caca7da7918a10 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:32 +0000
Subject: [PATCH 136/151] arm64/sysreg: Convert MVFR1_EL1 to automatic
generation
Convert MVFR1_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-34-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 10 ---------
arch/arm64/tools/sysreg | 39 +++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 561968f7b66d..d4b787d30fe8 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -170,7 +170,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
@@ -692,15 +691,6 @@
#define ID_DFR0_EL1_CopSDbg_SHIFT 4
#define ID_DFR0_EL1_CopDbg_SHIFT 0
-#define MVFR1_EL1_SIMDFMAC_SHIFT 28
-#define MVFR1_EL1_FPHP_SHIFT 24
-#define MVFR1_EL1_SIMDHP_SHIFT 20
-#define MVFR1_EL1_SIMDSP_SHIFT 16
-#define MVFR1_EL1_SIMDInt_SHIFT 12
-#define MVFR1_EL1_SIMDLS_SHIFT 8
-#define MVFR1_EL1_FPDNaN_SHIFT 4
-#define MVFR1_EL1_FPFtZ_SHIFT 0
-
#if defined(CONFIG_ARM64_4K_PAGES)
#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7a56a9a0efdf..3ca7e61265fd 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -645,6 +645,45 @@ Enum 3:0 SIMDReg
EndEnum
EndSysreg
+Sysreg MVFR1_EL1 3 0 0 3 1
+Res0 63:32
+Enum 31:28 SIMDFMAC
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 FPHP
+ 0b0000 NI
+ 0b0001 FPHP
+ 0b0010 FPHP_CONV
+ 0b0011 FP16
+EndEnum
+Enum 23:20 SIMDHP
+ 0b0000 NI
+ 0b0001 SIMDHP
+ 0b0001 SIMDHP_FLOAT
+EndEnum
+Enum 19:16 SIMDSP
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 SIMDInt
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 SIMDLS
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 7:4 FPDNaN
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 FPFtZ
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_PFR2_EL1 3 0 0 3 4
Res0 63:12
Enum 11:8 RAS_frac
From f70a810e01b2ad7eaad3470be1cf4220249bb251 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:33 +0000
Subject: [PATCH 137/151] arm64/sysreg: Convert MVFR2_EL1 to automatic
generation
Convert MVFR2_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-35-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 5 -----
arch/arm64/tools/sysreg | 17 +++++++++++++++++
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index d4b787d30fe8..aabc6db300fa 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -170,8 +170,6 @@
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
-#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
-
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
@@ -708,9 +706,6 @@
#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
#endif
-#define MVFR2_EL1_FPMisc_SHIFT 4
-#define MVFR2_EL1_SIMDMisc_SHIFT 0
-
#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */
#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 3ca7e61265fd..f0481f626405 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -684,6 +684,23 @@ Enum 3:0 FPFtZ
EndEnum
EndSysreg
+Sysreg MVFR2_EL1 3 0 0 3 2
+Res0 63:8
+Enum 7:4 FPMisc
+ 0b0000 NI
+ 0b0001 FP
+ 0b0010 FP_DIRECTED_ROUNDING
+ 0b0011 FP_ROUNDING
+ 0b0100 FP_MAX_MIN
+EndEnum
+Enum 3:0 SIMDMisc
+ 0b0000 NI
+ 0b0001 SIMD_DIRECTED_ROUNDING
+ 0b0010 SIMD_ROUNDING
+ 0b0011 SIMD_MAX_MIN
+EndEnum
+EndSysreg
+
Sysreg ID_PFR2_EL1 3 0 0 3 4
Res0 63:12
Enum 11:8 RAS_frac
From 8a950efa1ff0393967d79428b16af88193591ab3 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:34 +0000
Subject: [PATCH 138/151] arm64/sysreg: Convert ID_MMFR5_EL1 to automatic
generation
Convert ID_MMFR5_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-36-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 3 ---
arch/arm64/tools/sysreg | 12 ++++++++++++
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index aabc6db300fa..25e96aa84ae8 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -168,7 +168,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
-#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
@@ -679,8 +678,6 @@
#define ID_DFR1_EL1_MTPMU_SHIFT 0
-#define ID_MMFR5_EL1_ETS_SHIFT 0
-
#define ID_DFR0_EL1_PerfMon_SHIFT 24
#define ID_DFR0_EL1_MProfDbg_SHIFT 20
#define ID_DFR0_EL1_MMapTrc_SHIFT 16
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index f0481f626405..e83816c175a4 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -717,6 +717,18 @@ Enum 3:0 CSV3
EndEnum
EndSysreg
+Sysreg ID_MMFR5_EL1 3 0 0 3 6
+Res0 63:8
+Enum 7:4 nTLBPA
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 ETS
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+EndSysreg
+
Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
Enum 63:60 CSV3
0b0000 NI
From 58e010516ee63b04863de7030858670d1fc93471 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:35 +0000
Subject: [PATCH 139/151] arm64/sysreg: Convert ID_AFR0_EL1 to automatic
generation
Convert ID_AFR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-37-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 1 -
arch/arm64/tools/sysreg | 8 ++++++++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 25e96aa84ae8..c4d59ea7147a 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -167,7 +167,6 @@
#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
-#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index e83816c175a4..dd853ce2226d 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -127,6 +127,14 @@ Enum 3:0 ProgMod
EndEnum
EndSysreg
+Sysreg ID_AFR0_EL1 3 0 0 1 3
+Res0 63:16
+Field 15:12 IMPDEF3
+Field 11:8 IMPDEF2
+Field 7:4 IMPDEF1
+Field 3:0 IMPDEF0
+EndSysreg
+
Sysreg ID_MMFR0_EL1 3 0 0 1 4
Res0 63:32
Enum 31:28 InnerShr
From d044a9fbace769c0f47b52ab8bc39f69e8a6e922 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:36 +0000
Subject: [PATCH 140/151] arm64/sysreg: Convert ID_DFR0_EL1 to automatic
generation
Convert ID_DFR0_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221130171637.718182-38-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 14 ---------
arch/arm64/tools/sysreg | 50 +++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index c4d59ea7147a..a20ab575891f 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -165,7 +165,6 @@
#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
-#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
@@ -670,21 +669,8 @@
#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
#endif
-#define ID_DFR0_EL1_PerfMon_PMUv3 0x3
-#define ID_DFR0_EL1_PerfMon_PMUv3p1 0x4
-#define ID_DFR0_EL1_PerfMon_PMUv3p4 0x5
-#define ID_DFR0_EL1_PerfMon_PMUv3p5 0x6
-
#define ID_DFR1_EL1_MTPMU_SHIFT 0
-#define ID_DFR0_EL1_PerfMon_SHIFT 24
-#define ID_DFR0_EL1_MProfDbg_SHIFT 20
-#define ID_DFR0_EL1_MMapTrc_SHIFT 16
-#define ID_DFR0_EL1_CopTrc_SHIFT 12
-#define ID_DFR0_EL1_MMapDbg_SHIFT 8
-#define ID_DFR0_EL1_CopSDbg_SHIFT 4
-#define ID_DFR0_EL1_CopDbg_SHIFT 0
-
#if defined(CONFIG_ARM64_4K_PAGES)
#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index dd853ce2226d..6ff68bc7e776 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -127,6 +127,56 @@ Enum 3:0 ProgMod
EndEnum
EndSysreg
+Sysreg ID_DFR0_EL1 3 0 0 1 2
+Res0 63:32
+Enum 31:28 TraceFilt
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 27:24 PerfMon
+ 0b0000 NI
+ 0b0001 PMUv1
+ 0b0010 PMUv2
+ 0b0011 PMUv3
+ 0b0100 PMUv3p1
+ 0b0101 PMUv3p4
+ 0b0110 PMUv3p5
+ 0b0111 PMUv3p7
+ 0b1000 PMUv3p8
+ 0b1111 IMPDEF
+EndEnum
+Enum 23:20 MProfDbg
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 19:16 MMapTrc
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 15:12 CopTrc
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 11:8 MMapDbg
+ 0b0000 NI
+ 0b0100 Armv7
+ 0b0101 Armv7p1
+EndEnum
+Field 7:4 CopSDbg
+Enum 3:0 CopDbg
+ 0b0000 NI
+ 0b0010 Armv6
+ 0b0011 Armv6p1
+ 0b0100 Armv7
+ 0b0101 Armv7p1
+ 0b0110 Armv8
+ 0b0111 VHE
+ 0b1000 Debugv8p2
+ 0b1001 Debugv8p4
+ 0b1010 Debugv8p8
+EndEnum
+EndSysreg
+
Sysreg ID_AFR0_EL1 3 0 0 1 3
Res0 63:16
Field 15:12 IMPDEF3
From fa057722978ee52f319670f4ff4f181ecfa87290 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Wed, 30 Nov 2022 17:16:37 +0000
Subject: [PATCH 141/151] arm64/sysreg: Convert ID_DFR1_EL1 to automatic
generation
Convert ID_DFR1_EL1 to be automatically generated as per DDI0487I.a,
no functional changes.
Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20221130171637.718182-39-james.morse@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 2 --
arch/arm64/tools/sysreg | 13 +++++++++++++
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index a20ab575891f..3350fcdbde08 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -165,8 +165,6 @@
#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
-#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
-
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 6ff68bc7e776..bd5fceb26c54 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -775,6 +775,19 @@ Enum 3:0 CSV3
EndEnum
EndSysreg
+Sysreg ID_DFR1_EL1 3 0 0 3 5
+Res0 63:8
+Enum 7:4 HPMN0
+ 0b0000 NI
+ 0b0001 IMP
+EndEnum
+Enum 3:0 MTPMU
+ 0b0000 IMPDEF
+ 0b0001 IMP
+ 0b1111 NI
+EndEnum
+EndSysreg
+
Sysreg ID_MMFR5_EL1 3 0 0 3 6
Res0 63:8
Enum 7:4 nTLBPA
From acb3f4bc2108dca531f81233383e21f0bdc02267 Mon Sep 17 00:00:00 2001
From: Will Deacon <will@kernel.org>
Date: Thu, 1 Dec 2022 17:31:12 +0000
Subject: [PATCH 142/151] arm64/sysreg: Remove duplicate definitions from
asm/sysreg.h
With the new-fangled generation of asm/sysreg-defs.h, some definitions
have ended up being duplicated between the two files.
Remove these duplicate definitions, and consolidate the naming for
GMID_EL1_BS_WIDTH.
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/sysreg.h | 6 ------
arch/arm64/lib/mte.S | 2 +-
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 3350fcdbde08..097488cee174 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -667,8 +667,6 @@
#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
#endif
-#define ID_DFR1_EL1_MTPMU_SHIFT 0
-
#if defined(CONFIG_ARM64_4K_PAGES)
#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
@@ -719,10 +717,6 @@
#define SYS_RGSR_EL1_SEED_SHIFT 8
#define SYS_RGSR_EL1_SEED_MASK 0xffffUL
-/* GMID_EL1 field definitions */
-#define GMID_EL1_BS_SHIFT 0
-#define GMID_EL1_BS_SIZE 4
-
/* TFSR{,E0}_EL1 bit definitions */
#define SYS_TFSR_EL1_TF0_SHIFT 0
#define SYS_TFSR_EL1_TF1_SHIFT 1
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index 1b7c93ae7e63..5018ac03b6bf 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -18,7 +18,7 @@
*/
.macro multitag_transfer_size, reg, tmp
mrs_s \reg, SYS_GMID_EL1
- ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_SIZE
+ ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
mov \tmp, #4
lsl \reg, \tmp, \reg
.endm
From 98102a2cb7860b4d8226d6c2996f068fb4da5ed5 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 29 Nov 2022 21:59:23 +0000
Subject: [PATCH 143/151] kselftest/arm64: Hold fp-stress children until
they're all spawned
At present fp-stress has a bit of a thundering herd problem since the
children it spawns start running immediately, meaning that they can start
starving the parent process of CPU before it has even started all the
children. This is much more severe on virtual platforms since they tend to
support far more SVE and SME vector lengths, be slower in general and for
some have issues with performance when simulating multiple CPUs.
We can mitigate this problem by having all the child processes block before
starting the test program, meaning that we at least have all the child
processes started before we start heavily using CPU. We still have the same
load issues while waiting for the actual stress test programs to start up
and produce output but they're at least all ready to go before that kicks
in, resulting in substantial reductions in overall runtime on some of the
severely affected systems. One test was showing about 20% improvement.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221129215926.442895-2-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 41 +++++++++++++++++++-
1 file changed, 40 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index 69ca4a5f7e6e..7c04f5001648 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -44,6 +44,8 @@ static bool terminate;
static void drain_output(bool flush);
+static int startup_pipe[2];
+
static int num_processors(void)
{
long nproc = sysconf(_SC_NPROCESSORS_CONF);
@@ -81,13 +83,37 @@ static void child_start(struct child_data *child, const char *program)
exit(EXIT_FAILURE);
}
+ /*
+ * Duplicate the read side of the startup pipe to
+ * FD 3 so we can close everything else.
+ */
+ ret = dup2(startup_pipe[0], 3);
+ if (ret == -1) {
+ fprintf(stderr, "dup2() %d\n", errno);
+ exit(EXIT_FAILURE);
+ }
+
/*
* Very dumb mechanism to clean open FDs other than
* stdio. We don't want O_CLOEXEC for the pipes...
*/
- for (i = 3; i < 8192; i++)
+ for (i = 4; i < 8192; i++)
close(i);
+ /*
+ * Read from the startup pipe, there should be no data
+ * and we should block until it is closed. We just
+ * carry on on error since this isn't super critical.
+ */
+ ret = read(3, &i, sizeof(i));
+ if (ret < 0)
+ fprintf(stderr, "read(startp pipe) failed: %s (%d)\n",
+ strerror(errno), errno);
+ if (ret > 0)
+ fprintf(stderr, "%d bytes of data on startup pipe\n",
+ ret);
+ close(3);
+
ret = execl(program, program, NULL);
fprintf(stderr, "execl(%s) failed: %d (%s)\n",
program, errno, strerror(errno));
@@ -467,6 +493,12 @@ int main(int argc, char **argv)
strerror(errno), ret);
epoll_fd = ret;
+ /* Create a pipe which children will block on before execing */
+ ret = pipe(startup_pipe);
+ if (ret != 0)
+ ksft_exit_fail_msg("Failed to create startup pipe: %s (%d)\n",
+ strerror(errno), errno);
+
/* Get signal handers ready before we start any children */
memset(&sa, 0, sizeof(sa));
sa.sa_sigaction = handle_exit_signal;
@@ -499,6 +531,13 @@ int main(int argc, char **argv)
}
}
+ /*
+ * All children started, close the startup pipe and let them
+ * run.
+ */
+ close(startup_pipe[0]);
+ close(startup_pipe[1]);
+
for (;;) {
/* Did we get a signal asking us to exit? */
if (terminate)
From 92145d88ce0b216c1b99aaac92fec1f6a7d6ddde Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 29 Nov 2022 21:59:24 +0000
Subject: [PATCH 144/151] kselftest/arm64: Don't drain output while spawning
children
Now we hold execution of the stress test programs until all children are
started there is no need to drain output while that is happening.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221129215926.442895-3-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 8 --------
1 file changed, 8 deletions(-)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index 7c04f5001648..b3bbfe8d9f56 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -42,8 +42,6 @@ static struct child_data *children;
static int num_children;
static bool terminate;
-static void drain_output(bool flush);
-
static int startup_pipe[2];
static int num_processors(void)
@@ -138,12 +136,6 @@ static void child_start(struct child_data *child, const char *program)
ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n",
child->name, strerror(errno), errno);
}
-
- /*
- * Keep output flowing during child startup so logs
- * are more timely, can help debugging.
- */
- drain_output(false);
}
}
From c4e8720f2eb0c7f59082f41ad73b82e5d3f19f69 Mon Sep 17 00:00:00 2001
From: Mark Brown <broonie@kernel.org>
Date: Tue, 29 Nov 2022 21:59:25 +0000
Subject: [PATCH 145/151] kselftest/arm64: Allow epoll_wait() to return more
than one result
When everything is starting up we are likely to have a lot of child
processes producing output at once. This means that we can reduce
overhead a bit by allowing epoll_wait() to return more than one
descriptor at once, it cuts down on the number of system calls we need
to do which on virtual platforms where the syscall overhead is a bit
more noticable and we're likely to have a lot more children active can
make a small but noticable difference.
On physical platforms the relatively small number of processes being run
and vastly improved speeds push the effects of this change into the
noise.
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20221129215926.442895-4-broonie@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
tools/testing/selftests/arm64/fp/fp-stress.c | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
index b3bbfe8d9f56..f8b2f41aac36 100644
--- a/tools/testing/selftests/arm64/fp/fp-stress.c
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -39,6 +39,8 @@ struct child_data {
static int epoll_fd;
static struct child_data *children;
+static struct epoll_event *evs;
+static int tests;
static int num_children;
static bool terminate;
@@ -393,11 +395,11 @@ static void probe_vls(int vls[], int *vl_count, int set_vl)
/* Handle any pending output without blocking */
static void drain_output(bool flush)
{
- struct epoll_event ev;
int ret = 1;
+ int i;
while (ret > 0) {
- ret = epoll_wait(epoll_fd, &ev, 1, 0);
+ ret = epoll_wait(epoll_fd, evs, tests, 0);
if (ret < 0) {
if (errno == EINTR)
continue;
@@ -405,8 +407,8 @@ static void drain_output(bool flush)
strerror(errno), errno);
}
- if (ret == 1)
- child_output(ev.data.ptr, ev.events, flush);
+ for (i = 0; i < ret; i++)
+ child_output(evs[i].data.ptr, evs[i].events, flush);
}
}
@@ -419,12 +421,11 @@ int main(int argc, char **argv)
{
int ret;
int timeout = 10;
- int cpus, tests, i, j, c;
+ int cpus, i, j, c;
int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
bool all_children_started = false;
int seen_children;
int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
- struct epoll_event ev;
struct sigaction sa;
while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) {
@@ -510,6 +511,11 @@ int main(int argc, char **argv)
ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n",
strerror(errno), errno);
+ evs = calloc(tests, sizeof(*evs));
+ if (!evs)
+ ksft_exit_fail_msg("Failed to allocated %d epoll events\n",
+ tests);
+
for (i = 0; i < cpus; i++) {
for (j = 0; j < fpsimd_per_cpu; j++)
start_fpsimd(&children[num_children++], i, j);
@@ -543,7 +549,7 @@ int main(int argc, char **argv)
* useful in emulation where we will both be slow and
* likely to have a large set of VLs.
*/
- ret = epoll_wait(epoll_fd, &ev, 1, 1000);
+ ret = epoll_wait(epoll_fd, evs, tests, 1000);
if (ret < 0) {
if (errno == EINTR)
continue;
@@ -552,8 +558,11 @@ int main(int argc, char **argv)
}
/* Output? */
- if (ret == 1) {
- child_output(ev.data.ptr, ev.events, false);
+ if (ret > 0) {
+ for (i = 0; i < ret; i++) {
+ child_output(evs[i].data.ptr, evs[i].events,
+ false);
+ }
continue;
}
From 4361251cef466839795691e2628285e3f5093a98 Mon Sep 17 00:00:00 2001
From: Anshuman Khandual <anshuman.khandual@arm.com>
Date: Fri, 2 Dec 2022 07:26:11 +0530
Subject: [PATCH 146/151] arm_pmu: Drop redundant armpmu->map_event() in
armpmu_event_init()
__hw_perf_event_init() already calls armpmu->map_event() callback, and also
returns its error code including -ENOENT, along with a debug callout. Hence
an additional armpmu->map_event() check for -ENOENT is redundant.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Acked-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lore.kernel.org/r/20221202015611.338499-1-anshuman.khandual@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
---
drivers/perf/arm_pmu.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 3f07df5a7e95..ed89748b1612 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -514,9 +514,6 @@ static int armpmu_event_init(struct perf_event *event)
if (has_branch_stack(event))
return -EOPNOTSUPP;
- if (armpmu->map_event(event) == -ENOENT)
- return -ENOENT;
-
return __hw_perf_event_init(event);
}
From 67bc5b2d6d4802f46eceda246eb53a672b961831 Mon Sep 17 00:00:00 2001
From: Jisheng Zhang <jszhang@kernel.org>
Date: Sat, 3 Dec 2022 00:18:59 +0800
Subject: [PATCH 147/151] arm64: alternatives: add __init/__initconst to some
functions/variables
apply_alternatives_vdso(), __apply_alternatives_multi_stop() and
kernel_alternatives are not needed after booting, so mark the two
functions as __init and the var as __initconst.
Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Link: https://lore.kernel.org/r/20221202161859.2228-1-jszhang@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/alternative.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 0f5eaa4c3a39..d32d4ed5519b 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -196,7 +196,7 @@ static void __apply_alternatives(const struct alt_region *region,
}
}
-static void apply_alternatives_vdso(void)
+static void __init apply_alternatives_vdso(void)
{
struct alt_region region;
const struct elf64_hdr *hdr;
@@ -220,7 +220,7 @@ static void apply_alternatives_vdso(void)
__apply_alternatives(®ion, false, &all_capabilities[0]);
}
-static const struct alt_region kernel_alternatives = {
+static const struct alt_region kernel_alternatives __initconst = {
.begin = (struct alt_instr *)__alt_instructions,
.end = (struct alt_instr *)__alt_instructions_end,
};
@@ -229,7 +229,7 @@ static const struct alt_region kernel_alternatives = {
* We might be patching the stop_machine state machine, so implement a
* really simple polling protocol here.
*/
-static int __apply_alternatives_multi_stop(void *unused)
+static int __init __apply_alternatives_multi_stop(void *unused)
{
/* We always have a CPU 0 at this point (__init) */
if (smp_processor_id()) {
From 60f07e22a73d318cddaafa5ef41a10476807cc07 Mon Sep 17 00:00:00 2001
From: junhua huang <huang.junhua@zte.com.cn>
Date: Fri, 2 Dec 2022 15:11:10 +0800
Subject: [PATCH 148/151] arm64:uprobe fix the uprobe SWBP_INSN in big-endian
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We use uprobe in aarch64_be, which we found the tracee task would exit
due to SIGILL when we enable the uprobe trace.
We can see the replace inst from uprobe is not correct in aarch big-endian.
As in Armv8-A, instruction fetches are always treated as little-endian,
we should treat the UPROBE_SWBP_INSN as little-endian。
The test case is as following。
bash-4.4# ./mqueue_test_aarchbe 1 1 2 1 10 > /dev/null &
bash-4.4# cd /sys/kernel/debug/tracing/
bash-4.4# echo 'p:test /mqueue_test_aarchbe:0xc30 %x0 %x1' > uprobe_events
bash-4.4# echo 1 > events/uprobes/enable
bash-4.4#
bash-4.4# ps
PID TTY TIME CMD
140 ? 00:00:01 bash
237 ? 00:00:00 ps
[1]+ Illegal instruction ./mqueue_test_aarchbe 1 1 2 1 100 > /dev/null
which we debug use gdb as following:
bash-4.4# gdb attach 155
(gdb) disassemble send
Dump of assembler code for function send:
0x0000000000400c30 <+0>: .inst 0xa00020d4 ; undefined
0x0000000000400c34 <+4>: mov x29, sp
0x0000000000400c38 <+8>: str w0, [sp, #28]
0x0000000000400c3c <+12>: strb w1, [sp, #27]
0x0000000000400c40 <+16>: str xzr, [sp, #40]
0x0000000000400c44 <+20>: str xzr, [sp, #48]
0x0000000000400c48 <+24>: add x0, sp, #0x1b
0x0000000000400c4c <+28>: mov w3, #0x0 // #0
0x0000000000400c50 <+32>: mov x2, #0x1 // #1
0x0000000000400c54 <+36>: mov x1, x0
0x0000000000400c58 <+40>: ldr w0, [sp, #28]
0x0000000000400c5c <+44>: bl 0x405e10 <mq_send>
0x0000000000400c60 <+48>: str w0, [sp, #60]
0x0000000000400c64 <+52>: ldr w0, [sp, #60]
0x0000000000400c68 <+56>: ldp x29, x30, [sp], #64
0x0000000000400c6c <+60>: ret
End of assembler dump.
(gdb) info b
No breakpoints or watchpoints.
(gdb) c
Continuing.
Program received signal SIGILL, Illegal instruction.
0x0000000000400c30 in send ()
(gdb) x/10x 0x400c30
0x400c30 <send>: 0xd42000a0 0xfd030091 0xe01f00b9 0xe16f0039
0x400c40 <send+16>: 0xff1700f9 0xff1b00f9 0xe06f0091 0x03008052
0x400c50 <send+32>: 0x220080d2 0xe10300aa
(gdb) disassemble 0x400c30
Dump of assembler code for function send:
=> 0x0000000000400c30 <+0>: .inst 0xa00020d4 ; undefined
0x0000000000400c34 <+4>: mov x29, sp
0x0000000000400c38 <+8>: str w0, [sp, #28]
0x0000000000400c3c <+12>: strb w1, [sp, #27]
0x0000000000400c40 <+16>: str xzr, [sp, #40]
Signed-off-by: junhua huang <huang.junhua@zte.com.cn>
Link: https://lore.kernel.org/r/202212021511106844809@zte.com.cn
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/uprobes.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/uprobes.h b/arch/arm64/include/asm/uprobes.h
index 315eef654e39..ba4bff5ca674 100644
--- a/arch/arm64/include/asm/uprobes.h
+++ b/arch/arm64/include/asm/uprobes.h
@@ -12,7 +12,7 @@
#define MAX_UINSN_BYTES AARCH64_INSN_SIZE
-#define UPROBE_SWBP_INSN BRK64_OPCODE_UPROBES
+#define UPROBE_SWBP_INSN cpu_to_le32(BRK64_OPCODE_UPROBES)
#define UPROBE_SWBP_INSN_SIZE AARCH64_INSN_SIZE
#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES
From 0fbcd8abf3375052cc7627cc53aba6f2eb189fbb Mon Sep 17 00:00:00 2001
From: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Date: Fri, 2 Dec 2022 11:18:33 +0900
Subject: [PATCH 149/151] arm64: Prohibit instrumentation on arch_stack_walk()
Mark arch_stack_walk() as noinstr instead of notrace and inline functions
called from arch_stack_walk() as __always_inline so that user does not
put any instrumentations on it, because this function can be used from
return_address() which is used by lockdep.
Without this, if the kernel built with CONFIG_LOCKDEP=y, just probing
arch_stack_walk() via <tracefs>/kprobe_events will crash the kernel on
arm64.
# echo p arch_stack_walk >> ${TRACEFS}/kprobe_events
# echo 1 > ${TRACEFS}/events/kprobes/enable
kprobes: Failed to recover from reentered kprobes.
kprobes: Dump kprobe:
.symbol_name = arch_stack_walk, .offset = 0, .addr = arch_stack_walk+0x0/0x1c0
------------[ cut here ]------------
kernel BUG at arch/arm64/kernel/probes/kprobes.c:241!
kprobes: Failed to recover from reentered kprobes.
kprobes: Dump kprobe:
.symbol_name = arch_stack_walk, .offset = 0, .addr = arch_stack_walk+0x0/0x1c0
------------[ cut here ]------------
kernel BUG at arch/arm64/kernel/probes/kprobes.c:241!
PREEMPT SMP
Modules linked in:
CPU: 0 PID: 17 Comm: migration/0 Tainted: G N 6.1.0-rc5+ #6
Hardware name: linux,dummy-virt (DT)
Stopper: 0x0 <- 0x0
pstate: 600003c5 (nZCv DAIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : kprobe_breakpoint_handler+0x178/0x17c
lr : kprobe_breakpoint_handler+0x178/0x17c
sp : ffff8000080d3090
x29: ffff8000080d3090 x28: ffff0df5845798c0 x27: ffffc4f59057a774
x26: ffff0df5ffbba770 x25: ffff0df58f420f18 x24: ffff49006f641000
x23: ffffc4f590579768 x22: ffff0df58f420f18 x21: ffff8000080d31c0
x20: ffffc4f590579768 x19: ffffc4f590579770 x18: 0000000000000006
x17: 5f6b636174735f68 x16: 637261203d207264 x15: 64612e202c30203d
x14: 2074657366666f2e x13: 30633178302f3078 x12: 302b6b6c61775f6b
x11: 636174735f686372 x10: ffffc4f590dc5bd8 x9 : ffffc4f58eb31958
x8 : 00000000ffffefff x7 : ffffc4f590dc5bd8 x6 : 80000000fffff000
x5 : 000000000000bff4 x4 : 0000000000000000 x3 : 0000000000000000
x2 : 0000000000000000 x1 : ffff0df5845798c0 x0 : 0000000000000064
Call trace:
kprobes: Failed to recover from reentered kprobes.
kprobes: Dump kprobe:
.symbol_name = arch_stack_walk, .offset = 0, .addr = arch_stack_walk+0x0/0x1c0
------------[ cut here ]------------
kernel BUG at arch/arm64/kernel/probes/kprobes.c:241!
Fixes: 39ef362d2d45 ("arm64: Make return_address() use arch_stack_walk()")
Cc: stable@vger.kernel.org
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/166994751368.439920.3236636557520824664.stgit@devnote3
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/stacktrace.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 634279b3b03d..117e2c180f3c 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -23,8 +23,8 @@
*
* The regs must be on a stack currently owned by the calling task.
*/
-static inline void unwind_init_from_regs(struct unwind_state *state,
- struct pt_regs *regs)
+static __always_inline void unwind_init_from_regs(struct unwind_state *state,
+ struct pt_regs *regs)
{
unwind_init_common(state, current);
@@ -58,8 +58,8 @@ static __always_inline void unwind_init_from_caller(struct unwind_state *state)
* duration of the unwind, or the unwind will be bogus. It is never valid to
* call this for the current task.
*/
-static inline void unwind_init_from_task(struct unwind_state *state,
- struct task_struct *task)
+static __always_inline void unwind_init_from_task(struct unwind_state *state,
+ struct task_struct *task)
{
unwind_init_common(state, task);
@@ -186,7 +186,7 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
: stackinfo_get_unknown(); \
})
-noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry,
+noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
void *cookie, struct task_struct *task,
struct pt_regs *regs)
{
From 30a421552373d2becb378282408a399702a0f00a Mon Sep 17 00:00:00 2001
From: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Date: Fri, 2 Dec 2022 11:18:42 +0900
Subject: [PATCH 150/151] arm64: kprobes: Let arch do_page_fault() fix up page
fault in user handler
Since arm64's do_page_fault() can handle the page fault correctly
than kprobe_fault_handler() according to the context, let it handle
the page fault instead of simply call fixup_exception() in the
kprobe_fault_handler().
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/166994752269.439920.4801339965959400456.stgit@devnote3
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/probes/kprobes.c | 8 --------
1 file changed, 8 deletions(-)
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index c9e4d0720285..d2ae37f89774 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -294,14 +294,6 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
}
break;
- case KPROBE_HIT_ACTIVE:
- case KPROBE_HIT_SSDONE:
- /*
- * In case the user-specified fault handler returned
- * zero, try to fix up.
- */
- if (fixup_exception(regs))
- return 1;
}
return 0;
}
From 3b84efc066f8793a6581018d92da9b304abd8ce8 Mon Sep 17 00:00:00 2001
From: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Date: Fri, 2 Dec 2022 11:18:52 +0900
Subject: [PATCH 151/151] arm64: kprobes: Return DBG_HOOK_ERROR if kprobes can
not handle a BRK
Return DBG_HOOK_ERROR if kprobes can not handle a BRK because it
fails to find a kprobe corresponding to the address.
Since arm64 kprobes uses stop_machine based text patching for removing
BRK, it ensures all running kprobe_break_handler() is done at that point.
And after removing the BRK, it removes the kprobe from its hash list.
Thus, if the kprobe_break_handler() fails to find kprobe from hash list,
there is a bug.
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/166994753273.439920.6629626290560350760.stgit@devnote3
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/probes/kprobes.c | 80 ++++++++++++++----------------
1 file changed, 37 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index d2ae37f89774..f35d059a9a36 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -298,7 +298,8 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
return 0;
}
-static void __kprobes kprobe_handler(struct pt_regs *regs)
+static int __kprobes
+kprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr)
{
struct kprobe *p, *cur_kprobe;
struct kprobe_ctlblk *kcb;
@@ -308,39 +309,44 @@ static void __kprobes kprobe_handler(struct pt_regs *regs)
cur_kprobe = kprobe_running();
p = get_kprobe((kprobe_opcode_t *) addr);
-
- if (p) {
- if (cur_kprobe) {
- if (reenter_kprobe(p, regs, kcb))
- return;
- } else {
- /* Probe hit */
- set_current_kprobe(p);
- kcb->kprobe_status = KPROBE_HIT_ACTIVE;
-
- /*
- * If we have no pre-handler or it returned 0, we
- * continue with normal processing. If we have a
- * pre-handler and it returned non-zero, it will
- * modify the execution path and no need to single
- * stepping. Let's just reset current kprobe and exit.
- */
- if (!p->pre_handler || !p->pre_handler(p, regs)) {
- setup_singlestep(p, regs, kcb, 0);
- } else
- reset_current_kprobe();
- }
+ if (WARN_ON_ONCE(!p)) {
+ /*
+ * Something went wrong. This BRK used an immediate reserved
+ * for kprobes, but we couldn't find any corresponding probe.
+ */
+ return DBG_HOOK_ERROR;
}
- /*
- * The breakpoint instruction was removed right
- * after we hit it. Another cpu has removed
- * either a probepoint or a debugger breakpoint
- * at this address. In either case, no further
- * handling of this interrupt is appropriate.
- * Return back to original instruction, and continue.
- */
+
+ if (cur_kprobe) {
+ /* Hit a kprobe inside another kprobe */
+ if (!reenter_kprobe(p, regs, kcb))
+ return DBG_HOOK_ERROR;
+ } else {
+ /* Probe hit */
+ set_current_kprobe(p);
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+
+ /*
+ * If we have no pre-handler or it returned 0, we
+ * continue with normal processing. If we have a
+ * pre-handler and it returned non-zero, it will
+ * modify the execution path and not need to single-step
+ * Let's just reset current kprobe and exit.
+ */
+ if (!p->pre_handler || !p->pre_handler(p, regs))
+ setup_singlestep(p, regs, kcb, 0);
+ else
+ reset_current_kprobe();
+ }
+
+ return DBG_HOOK_HANDLED;
}
+static struct break_hook kprobes_break_hook = {
+ .imm = KPROBES_BRK_IMM,
+ .fn = kprobe_breakpoint_handler,
+};
+
static int __kprobes
kprobe_breakpoint_ss_handler(struct pt_regs *regs, unsigned long esr)
{
@@ -365,18 +371,6 @@ static struct break_hook kprobes_break_ss_hook = {
.fn = kprobe_breakpoint_ss_handler,
};
-static int __kprobes
-kprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr)
-{
- kprobe_handler(regs);
- return DBG_HOOK_HANDLED;
-}
-
-static struct break_hook kprobes_break_hook = {
- .imm = KPROBES_BRK_IMM,
- .fn = kprobe_breakpoint_handler,
-};
-
/*
* Provide a blacklist of symbols identifying ranges which cannot be kprobed.
* This blacklist is exposed to userspace via debugfs (kprobes/blacklist).