From cad6431b867507779c41b00baaf18382467ef0a0 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 3 Oct 2023 13:25:26 +0200 Subject: [PATCH 1/5] mlxsw: Mark high entropy key blocks For 12 key blocks in the A-TCAM, rules are split into two records, which constitute two lookups. The two records are linked using a "large entry key ID". Due to a Spectrum-4 hardware issue, KVD entries that correspond to key blocks 0 to 5 of 12 key blocks A-TCAM entries will be placed in the same KVD pipe if they only differ in their "large entry key ID", as it is ignored. This results in a reduced scale. To reduce the probability of this issue, we can place key blocks with high entropy in blocks 0 to 5. The idea is to place blocks that are changed often in blocks 0 to 5, for example, key blocks that match on IPv4 addresses or the LSBs of IPv6 addresses. Such placement will reduce the probability of these blocks to be same. Mark several blocks with 'high_entropy' flag, so later we will take into account this flag and place them in blocks 0 to 5. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h | 9 +++++++++ .../ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c | 12 ++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h index 1c76aa3ffab7..98a05598178b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h @@ -119,6 +119,7 @@ struct mlxsw_afk_block { u16 encoding; /* block ID */ struct mlxsw_afk_element_inst *instances; unsigned int instances_count; + bool high_entropy; }; #define MLXSW_AFK_BLOCK(_encoding, _instances) \ @@ -128,6 +129,14 @@ struct mlxsw_afk_block { .instances_count = ARRAY_SIZE(_instances), \ } +#define MLXSW_AFK_BLOCK_HIGH_ENTROPY(_encoding, _instances) \ + { \ + .encoding = _encoding, \ + .instances = _instances, \ + .instances_count = ARRAY_SIZE(_instances), \ + .high_entropy = true, \ + } + struct mlxsw_afk_element_usage { DECLARE_BITMAP(usage, MLXSW_AFK_ELEMENT_MAX); }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c index 4b3564f5fd65..eaad78605602 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c @@ -334,14 +334,14 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv6_2b[] = { }; static const struct mlxsw_afk_block mlxsw_sp4_afk_blocks[] = { - MLXSW_AFK_BLOCK(0x10, mlxsw_sp_afk_element_info_mac_0), - MLXSW_AFK_BLOCK(0x11, mlxsw_sp_afk_element_info_mac_1), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x10, mlxsw_sp_afk_element_info_mac_0), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x11, mlxsw_sp_afk_element_info_mac_1), MLXSW_AFK_BLOCK(0x12, mlxsw_sp_afk_element_info_mac_2), MLXSW_AFK_BLOCK(0x13, mlxsw_sp_afk_element_info_mac_3), MLXSW_AFK_BLOCK(0x14, mlxsw_sp_afk_element_info_mac_4), - MLXSW_AFK_BLOCK(0x1A, mlxsw_sp_afk_element_info_mac_5b), - MLXSW_AFK_BLOCK(0x38, mlxsw_sp_afk_element_info_ipv4_0), - MLXSW_AFK_BLOCK(0x39, mlxsw_sp_afk_element_info_ipv4_1), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x1A, mlxsw_sp_afk_element_info_mac_5b), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x38, mlxsw_sp_afk_element_info_ipv4_0), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x39, mlxsw_sp_afk_element_info_ipv4_1), MLXSW_AFK_BLOCK(0x3A, mlxsw_sp_afk_element_info_ipv4_2), MLXSW_AFK_BLOCK(0x36, mlxsw_sp_afk_element_info_ipv4_5b), MLXSW_AFK_BLOCK(0x40, mlxsw_sp_afk_element_info_ipv6_0), @@ -350,7 +350,7 @@ static const struct mlxsw_afk_block mlxsw_sp4_afk_blocks[] = { MLXSW_AFK_BLOCK(0x43, mlxsw_sp_afk_element_info_ipv6_3), MLXSW_AFK_BLOCK(0x44, mlxsw_sp_afk_element_info_ipv6_4), MLXSW_AFK_BLOCK(0x45, mlxsw_sp_afk_element_info_ipv6_5), - MLXSW_AFK_BLOCK(0x90, mlxsw_sp_afk_element_info_l4_0), + MLXSW_AFK_BLOCK_HIGH_ENTROPY(0x90, mlxsw_sp_afk_element_info_l4_0), MLXSW_AFK_BLOCK(0x92, mlxsw_sp_afk_element_info_l4_2), }; From 0a67b7a0ec36bda6ce2898016415b36d3b85dca8 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 3 Oct 2023 13:25:27 +0200 Subject: [PATCH 2/5] mlxsw: core_acl_flex_keys: Add a bitmap to save which blocks are chosen Currently, mlxsw_afk_picker() chooses which blocks will be used for a given list of elements, and fills the blocks during the searching - when a key block is found with most hits, it adds it and removes the elements from the count of hits. This should be changed as we want to be able to choose which blocks will be placed in blocks 0 to 5. To separate between choosing blocks and filling blocks, several pre-changes are required. The indexes of the chosen blocks should be saved, so then the relevant blocks will be filled at the end of search. Allocate a bitmap for chosen blocks, when a block is found with most hits, set the relevant bit in the bitmap. This bitmap will be used in a following patch. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c index 745438d8ae10..e0b4834700dd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c @@ -225,6 +225,7 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, struct mlxsw_afk_element_usage *elusage) { struct mlxsw_afk_picker *picker; + unsigned long *chosen_blocks_bm; enum mlxsw_afk_element element; int err; @@ -232,6 +233,12 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, if (!picker) return -ENOMEM; + chosen_blocks_bm = bitmap_zalloc(mlxsw_afk->blocks_count, GFP_KERNEL); + if (!chosen_blocks_bm) { + err = -ENOMEM; + goto err_bitmap_alloc; + } + /* Since the same elements could be present in multiple blocks, * we must find out optimal block list in order to make the * block count as low as possible. @@ -256,6 +263,9 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, err = block_index; goto out; } + + __set_bit(block_index, chosen_blocks_bm); + err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, block_index, key_info); if (err) @@ -265,6 +275,8 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, err = 0; out: + bitmap_free(chosen_blocks_bm); +err_bitmap_alloc: kfree(picker); return err; } From 545535fd30dc8eaac96c09d774e8ba57bb247267 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 3 Oct 2023 13:25:28 +0200 Subject: [PATCH 3/5] mlxsw: core_acl_flex_keys: Save chosen elements per block Currently, mlxsw_afk_picker() chooses which blocks will be used for a given list of elements, and fills the blocks during the searching - when a key block is found with most hits, it adds it and removes the elements from the count of hits. This should be changed as we want to be able to choose which blocks will be placed in blocks 0 to 5. To separate between choosing blocks and filling blocks, several pre-changes are required. During the search, the structure 'mlxsw_afk_picker' is used per block, it contains how many elements from the required list appear in the block. When a block is chosen and filled, this bitmap of elements is cleaned. To be able to fill the blocks at the end, add a bitmap called 'chosen_element' as part of picker. When a block is chosen, copy the 'element' bitmap to it. Use the new bitmap as part of mlxsw_afk_picker_key_info_add(). So later, when filling the block will be done at the end of the searching, we will use the copied bitmap that contains the elements that should be used in the block. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c index e0b4834700dd..7679df860a74 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c @@ -138,6 +138,7 @@ mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, struct mlxsw_afk_picker { DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); + DECLARE_BITMAP(chosen_element, MLXSW_AFK_ELEMENT_MAX); unsigned int total; }; @@ -208,7 +209,7 @@ static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, if (key_info->blocks_count == mlxsw_afk->max_blocks) return -EINVAL; - for_each_set_bit(element, picker[block_index].element, + for_each_set_bit(element, picker[block_index].chosen_element, MLXSW_AFK_ELEMENT_MAX) { key_info->element_to_block[element] = key_info->blocks_count; mlxsw_afk_element_usage_add(&key_info->elusage, element); @@ -266,6 +267,9 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, __set_bit(block_index, chosen_blocks_bm); + bitmap_copy(picker[block_index].chosen_element, + picker[block_index].element, MLXSW_AFK_ELEMENT_MAX); + err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, block_index, key_info); if (err) From 900f4285bbc2661e330036ee0fae046c7b79036b Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 3 Oct 2023 13:25:29 +0200 Subject: [PATCH 4/5] mlxsw: core_acl_flex_keys: Save chosen elements in all blocks per search Currently, mlxsw_afk_picker() chooses which blocks will be used for a given list of elements, and fills the blocks during the searching - when a key block is found with most hits, it adds it and removes the elements from the count of hits. This should be changed as we want to be able to choose which blocks will be placed in blocks 0 to 5. To separate between choosing blocks and filling blocks, several pre-changes are required. Currently, the indication of whether all elements were found in the chosen blocks is by the structure 'key_info->elusage'. This structure is updated when block is filled as part of mlxsw_afk_picker_key_info_add(). A following patch will call this function only after choosing all the blocks. Add a bitmap called 'elusage_chosen' to store which elements were chosen in the chosen blocks. Change the condition in the loop to check elements that were chosen, not elements that were already filled in the blocks. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c index 7679df860a74..c0e0493f67b2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c @@ -225,6 +225,7 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, struct mlxsw_afk_key_info *key_info, struct mlxsw_afk_element_usage *elusage) { + DECLARE_BITMAP(elusage_chosen, MLXSW_AFK_ELEMENT_MAX) = {0}; struct mlxsw_afk_picker *picker; unsigned long *chosen_blocks_bm; enum mlxsw_afk_element element; @@ -270,12 +271,18 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, bitmap_copy(picker[block_index].chosen_element, picker[block_index].element, MLXSW_AFK_ELEMENT_MAX); + bitmap_or(elusage_chosen, elusage_chosen, + picker[block_index].chosen_element, + MLXSW_AFK_ELEMENT_MAX); + err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, block_index, key_info); if (err) goto out; mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); - } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); + + } while (!bitmap_equal(elusage_chosen, elusage->usage, + MLXSW_AFK_ELEMENT_MAX)); err = 0; out: From c01e24936d16fcdac20849e2a71f52a631a995c5 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 3 Oct 2023 13:25:30 +0200 Subject: [PATCH 5/5] mlxsw: core_acl_flex_keys: Fill blocks with high entropy first The previous patches prepared the code to allow separating between choosing blocks and filling blocks. Do not add blocks as part of the loop that chooses them. When all the required blocks are set in the bitmap 'chosen_blocks_bm', start filling blocks. Iterate over the bitmap twice - first add only blocks that are marked with 'high_entropy' flag. Then, fill the rest of the blocks. The idea is to place key blocks with high entropy in blocks 0 to 5. See more details in previous patches. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: David S. Miller --- .../mellanox/mlxsw/core_acl_flex_keys.c | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c index c0e0493f67b2..0d5e6f9b466e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c @@ -221,6 +221,36 @@ static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, return 0; } +static int mlxsw_afk_keys_fill(struct mlxsw_afk *mlxsw_afk, + unsigned long *chosen_blocks_bm, + struct mlxsw_afk_picker *picker, + struct mlxsw_afk_key_info *key_info) +{ + int i, err; + + /* First fill only key blocks with high_entropy. */ + for_each_set_bit(i, chosen_blocks_bm, mlxsw_afk->blocks_count) { + if (!mlxsw_afk->blocks[i].high_entropy) + continue; + + err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, i, + key_info); + if (err) + return err; + __clear_bit(i, chosen_blocks_bm); + } + + /* Fill the rest of key blocks. */ + for_each_set_bit(i, chosen_blocks_bm, mlxsw_afk->blocks_count) { + err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, i, + key_info); + if (err) + return err; + } + + return 0; +} + static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, struct mlxsw_afk_key_info *key_info, struct mlxsw_afk_element_usage *elusage) @@ -275,16 +305,13 @@ static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, picker[block_index].chosen_element, MLXSW_AFK_ELEMENT_MAX); - err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, - block_index, key_info); - if (err) - goto out; mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); } while (!bitmap_equal(elusage_chosen, elusage->usage, MLXSW_AFK_ELEMENT_MAX)); - err = 0; + err = mlxsw_afk_keys_fill(mlxsw_afk, chosen_blocks_bm, picker, + key_info); out: bitmap_free(chosen_blocks_bm); err_bitmap_alloc: