iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kasan, arm64: implement HW_TAGS runtime

Browse Source 
Provide implementation of KASAN functions required for the hardware
tag-based mode.  Those include core functions for memory and pointer
tagging (tags_hw.c) and bug reporting (report_tags_hw.c).  Also adapt
common KASAN code to support the new mode.

Link: https://lkml.kernel.org/r/cfd0fbede579a6b66755c98c88c108e54f9c56bf.1606161801.git.andreyknvl@google.com
Signed-off-by: Andrey Konovalov <andreyknvl@google.com>
Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Alexander Potapenko <glider@google.com>
Tested-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Branislav Rankov <Branislav.Rankov@arm.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Marco Elver <elver@google.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Andrey Konovalov 2020-12-22 12:02:10 -08:00 committed by Linus Torvalds
parent 0fea6e9af8
commit 2e903b9147
14 changed files with 178 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
arch/arm64/include/asm/memory.h
View File

@ -214,7 +214,7 @@ static inline unsigned long kaslr_offset(void)
(__force __typeof__(addr))__addr; \
})
#ifdef CONFIG_KASAN_SW_TAGS
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
#define __tag_shifted(tag) ((u64)(tag) << 56)
#define __tag_reset(addr) __untagged_addr(addr)
#define __tag_get(addr) (__u8)((u64)(addr) >> 56)
@ -222,7 +222,7 @@ static inline unsigned long kaslr_offset(void)
#define __tag_shifted(tag) 0UL
#define __tag_reset(addr) (addr)
#define __tag_get(addr) 0
#endif /* CONFIG_KASAN_SW_TAGS */
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline const void *__tag_set(const void *addr, u8 tag)
{

3
arch/arm64/kernel/cpufeature.c
View File

@ -70,6 +70,7 @@
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/kasan.h>
#include <asm/cpu.h>
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
@ -1710,6 +1711,8 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
cleared_zero_page = true;
mte_clear_page_tags(lm_alias(empty_zero_page));
}
kasan_init_hw_tags_cpu();
}
#endif /* CONFIG_ARM64_MTE */

2
arch/arm64/kernel/smp.c
View File

@ -462,6 +462,8 @@ void __init smp_prepare_boot_cpu(void)
/* Conditionally switch to GIC PMR for interrupt masking */
if (system_uses_irq_prio_masking())
init_gic_priority_masking();
kasan_init_hw_tags();
}
static u64 __init of_get_cpu_mpidr(struct device_node *dn)

24
include/linux/kasan.h
View File

@ -190,25 +190,35 @@ static inline void kasan_record_aux_stack(void *ptr) {}
#endif /* CONFIG_KASAN_GENERIC */
#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
void *kasan_reset_tag(const void *addr);
bool kasan_report(unsigned long addr, size_t size,
bool is_write, unsigned long ip);
#else /* CONFIG_KASAN_SW_TAGS */
static inline void kasan_init_sw_tags(void) { }
#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline void *kasan_reset_tag(const void *addr)
{
return (void *)addr;
}
#endif /* CONFIG_KASAN_SW_TAGS */
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#else
static inline void kasan_init_sw_tags(void) { }
#endif
#ifdef CONFIG_KASAN_HW_TAGS
void kasan_init_hw_tags_cpu(void);
void __init kasan_init_hw_tags(void);
#else
static inline void kasan_init_hw_tags_cpu(void) { }
static inline void kasan_init_hw_tags(void) { }
#endif
#ifdef CONFIG_KASAN_VMALLOC

2
include/linux/mm.h
View File

@ -1421,7 +1421,7 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
}
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_KASAN_SW_TAGS
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
static inline u8 page_kasan_tag(const struct page *page)
{
return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;

2
include/linux/page-flags-layout.h
View File

@ -77,7 +77,7 @@
#define LAST_CPUPID_SHIFT 0
#endif
#ifdef CONFIG_KASAN_SW_TAGS
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
#define KASAN_TAG_WIDTH 8
#else
#define KASAN_TAG_WIDTH 0

5
mm/kasan/Makefile
View File

@ -10,8 +10,10 @@ CFLAGS_REMOVE_init.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_quarantine.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_report.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_report_generic.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_report_hw_tags.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_report_sw_tags.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_shadow.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_hw_tags.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_sw_tags.o = $(CC_FLAGS_FTRACE)
# Function splitter causes unnecessary splits in __asan_load1/__asan_store1
@ -27,10 +29,13 @@ CFLAGS_init.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_quarantine.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_report.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_report_generic.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_report_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_report_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_shadow.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
obj-$(CONFIG_KASAN) := common.o report.o
obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o
obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o
obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o

15
mm/kasan/common.c
View File

@ -118,7 +118,7 @@ void kasan_free_pages(struct page *page, unsigned int order)
*/
static inline unsigned int optimal_redzone(unsigned int object_size)
{
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
if (!IS_ENABLED(CONFIG_KASAN_GENERIC))
return 0;
return
@ -183,14 +183,14 @@ size_t kasan_metadata_size(struct kmem_cache *cache)
struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
const void *object)
{
return (void *)object + cache->kasan_info.alloc_meta_offset;
return (void *)reset_tag(object) + cache->kasan_info.alloc_meta_offset;
}
struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
const void *object)
{
BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
return (void *)object + cache->kasan_info.free_meta_offset;
return (void *)reset_tag(object) + cache->kasan_info.free_meta_offset;
}
void kasan_poison_slab(struct page *page)
@ -272,9 +272,8 @@ void * __must_check kasan_init_slab_obj(struct kmem_cache *cache,
alloc_info = get_alloc_info(cache, object);
__memset(alloc_info, 0, sizeof(*alloc_info));
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
object = set_tag(object,
assign_tag(cache, object, true, false));
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS))
object = set_tag(object, assign_tag(cache, object, true, false));
return (void *)object;
}
@ -342,10 +341,10 @@ static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object,
redzone_end = round_up((unsigned long)object + cache->object_size,
KASAN_GRANULE_SIZE);
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS))
tag = assign_tag(cache, object, false, keep_tag);
/* Tag is ignored in set_tag without CONFIG_KASAN_SW_TAGS */
/* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */
unpoison_range(set_tag(object, tag), size);
poison_range((void *)redzone_start, redzone_end - redzone_start,
KASAN_KMALLOC_REDZONE);

80
mm/kasan/hw_tags.c Normal file
View File

@ -0,0 +1,80 @@
// SPDX-License-Identifier: GPL-2.0
/*
* This file contains core hardware tag-based KASAN code.
*
* Copyright (c) 2020 Google, Inc.
* Author: Andrey Konovalov <andreyknvl@google.com>
*/
#define pr_fmt(fmt) "kasan: " fmt
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/types.h>
#include "kasan.h"
/* kasan_init_hw_tags_cpu() is called for each CPU. */
void kasan_init_hw_tags_cpu(void)
{
hw_init_tags(KASAN_TAG_MAX);
hw_enable_tagging();
}
/* kasan_init_hw_tags() is called once on boot CPU. */
void __init kasan_init_hw_tags(void)
{
pr_info("KernelAddressSanitizer initialized\n");
}
void *kasan_reset_tag(const void *addr)
{
return reset_tag(addr);
}
void poison_range(const void *address, size_t size, u8 value)
{
hw_set_mem_tag_range(reset_tag(address),
round_up(size, KASAN_GRANULE_SIZE), value);
}
void unpoison_range(const void *address, size_t size)
{
hw_set_mem_tag_range(reset_tag(address),
round_up(size, KASAN_GRANULE_SIZE), get_tag(address));
}
u8 random_tag(void)
{
return hw_get_random_tag();
}
bool check_invalid_free(void *addr)
{
u8 ptr_tag = get_tag(addr);
u8 mem_tag = hw_get_mem_tag(addr);
return (mem_tag == KASAN_TAG_INVALID) ||
(ptr_tag != KASAN_TAG_KERNEL && ptr_tag != mem_tag);
}
void kasan_set_free_info(struct kmem_cache *cache,
void *object, u8 tag)
{
struct kasan_alloc_meta *alloc_meta;
alloc_meta = get_alloc_info(cache, object);
kasan_set_track(&alloc_meta->free_track[0], GFP_NOWAIT);
}
struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
void *object, u8 tag)
{
struct kasan_alloc_meta *alloc_meta;
alloc_meta = get_alloc_info(cache, object);
return &alloc_meta->free_track[0];
}

19
mm/kasan/kasan.h
View File

@ -154,6 +154,11 @@ struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
const void *object);
void poison_range(const void *address, size_t size, u8 value);
void unpoison_range(const void *address, size_t size);
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
{
return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET)
@ -165,9 +170,6 @@ static inline bool addr_has_metadata(const void *addr)
return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START));
}
void poison_range(const void *address, size_t size, u8 value);
void unpoison_range(const void *address, size_t size);
/**
* check_memory_region - Check memory region, and report if invalid access.
* @addr: the accessed address
@ -179,6 +181,15 @@ void unpoison_range(const void *address, size_t size);
bool check_memory_region(unsigned long addr, size_t size, bool write,
unsigned long ret_ip);
#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
static inline bool addr_has_metadata(const void *addr)
{
return true;
}
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
bool check_invalid_free(void *addr);
void *find_first_bad_addr(void *addr, size_t size);
@ -215,7 +226,7 @@ static inline void quarantine_reduce(void) { }
static inline void quarantine_remove_cache(struct kmem_cache *cache) { }
#endif
#ifdef CONFIG_KASAN_SW_TAGS
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
void print_tags(u8 addr_tag, const void *addr);

42
mm/kasan/report_hw_tags.c Normal file
View File

@ -0,0 +1,42 @@
// SPDX-License-Identifier: GPL-2.0
/*
* This file contains hardware tag-based KASAN specific error reporting code.
*
* Copyright (c) 2020 Google, Inc.
* Author: Andrey Konovalov <andreyknvl@google.com>
*/
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/types.h>
#include "kasan.h"
const char *get_bug_type(struct kasan_access_info *info)
{
return "invalid-access";
}
void *find_first_bad_addr(void *addr, size_t size)
{
return reset_tag(addr);
}
void metadata_fetch_row(char *buffer, void *row)
{
int i;
for (i = 0; i < META_BYTES_PER_ROW; i++)
buffer[i] = hw_get_mem_tag(row + i * KASAN_GRANULE_SIZE);
}
void print_tags(u8 addr_tag, const void *addr)
{
u8 memory_tag = hw_get_mem_tag((void *)addr);
pr_err("Pointer tag: [%02x], memory tag: [%02x]\n",
addr_tag, memory_tag);
}

2
mm/kasan/report_sw_tags.c
View File

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/*
* This file contains tag-based KASAN specific error reporting code.
* This file contains software tag-based KASAN specific error reporting code.
*
* Copyright (c) 2014 Samsung Electronics Co., Ltd.
* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>

2
mm/kasan/shadow.c
View File

@ -107,7 +107,7 @@ void unpoison_range(const void *address, size_t size)
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
*shadow = tag;
else
else /* CONFIG_KASAN_GENERIC */
*shadow = size & KASAN_GRANULE_MASK;
}
}

2
mm/kasan/sw_tags.c
View File

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/*
* This file contains core tag-based KASAN code.
* This file contains core software tag-based KASAN code.
*
* Copyright (c) 2018 Google, Inc.
* Author: Andrey Konovalov <andreyknvl@google.com>