2011-10-26 06:26:31 +04:00
/*
2014-05-07 03:48:38 +04:00
* Copyright ( c ) 2007 - 2014 Nicira , Inc .
2011-10-26 06:26:31 +04:00
*
* This program is free software ; you can redistribute it and / or
* modify it under the terms of version 2 of the GNU General Public
* License as published by the Free Software Foundation .
*
* This program is distributed in the hope that it will be useful , but
* WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the GNU
* General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 51 Franklin Street , Fifth Floor , Boston , MA
* 02110 - 1301 , USA
*/
# ifndef FLOW_H
# define FLOW_H 1
2013-10-30 04:22:21 +04:00
# include <linux/cache.h>
2011-10-26 06:26:31 +04:00
# include <linux/kernel.h>
# include <linux/netlink.h>
# include <linux/openvswitch.h>
# include <linux/spinlock.h>
# include <linux/types.h>
# include <linux/rcupdate.h>
# include <linux/if_ether.h>
# include <linux/in6.h>
# include <linux/jiffies.h>
# include <linux/time.h>
# include <linux/flex_array.h>
# include <net/inet_ecn.h>
2015-07-21 11:43:54 +03:00
# include <net/ip_tunnels.h>
2015-07-21 11:44:03 +03:00
# include <net/dst_metadata.h>
2011-10-26 06:26:31 +04:00
struct sk_buff ;
2014-10-04 02:35:33 +04:00
/* Store options at the end of the array if they are less than the
* maximum size . This allows us to get the benefits of variable length
* matching for small options .
*/
2015-01-15 05:53:57 +03:00
# define TUN_METADATA_OFFSET(opt_len) \
( FIELD_SIZEOF ( struct sw_flow_key , tun_opts ) - opt_len )
# define TUN_METADATA_OPTS(flow_key, opt_len) \
( ( void * ) ( ( flow_key ) - > tun_opts + TUN_METADATA_OFFSET ( opt_len ) ) )
2014-10-04 02:35:33 +04:00
2015-07-21 11:44:03 +03:00
struct ovs_tunnel_info {
struct metadata_dst * tun_dst ;
} ;
2014-11-06 17:51:24 +03:00
# define OVS_SW_FLOW_KEY_METADATA_SIZE \
( offsetof ( struct sw_flow_key , recirc_id ) + \
FIELD_SIZEOF ( struct sw_flow_key , recirc_id ) )
2011-10-26 06:26:31 +04:00
struct sw_flow_key {
2014-10-04 02:35:33 +04:00
u8 tun_opts [ 255 ] ;
u8 tun_opts_len ;
2015-07-21 11:43:54 +03:00
struct ip_tunnel_key tun_key ; /* Encapsulating tunnel key. */
2011-10-26 06:26:31 +04:00
struct {
u32 priority ; /* Packet QoS priority. */
2012-11-26 23:24:11 +04:00
u32 skb_mark ; /* SKB mark. */
2012-08-23 23:40:54 +04:00
u16 in_port ; /* Input switch port (or DP_MAX_PORTS). */
2014-05-05 20:54:49 +04:00
} __packed phy ; /* Safe when right after 'tun_key'. */
2014-09-16 06:37:25 +04:00
u32 ovs_flow_hash ; /* Datapath computed hash value. */
u32 recirc_id ; /* Recirculation ID. */
2011-10-26 06:26:31 +04:00
struct {
u8 src [ ETH_ALEN ] ; /* Ethernet source address. */
u8 dst [ ETH_ALEN ] ; /* Ethernet destination address. */
__be16 tci ; /* 0 if no VLAN, VLAN_TAG_PRESENT set otherwise. */
__be16 type ; /* Ethernet frame type. */
} eth ;
2014-10-06 16:05:13 +04:00
union {
struct {
__be32 top_lse ; /* top label stack entry */
} mpls ;
struct {
u8 proto ; /* IP protocol or lower 8 bits of ARP opcode. */
u8 tos ; /* IP ToS. */
u8 ttl ; /* IP TTL/hop limit. */
u8 frag ; /* One of OVS_FRAG_TYPE_*. */
} ip ;
} ;
2014-05-05 20:54:49 +04:00
struct {
__be16 src ; /* TCP/UDP/SCTP source port. */
__be16 dst ; /* TCP/UDP/SCTP destination port. */
__be16 flags ; /* TCP flags. */
} tp ;
2011-10-26 06:26:31 +04:00
union {
struct {
struct {
__be32 src ; /* IP source address. */
__be32 dst ; /* IP destination address. */
} addr ;
2014-05-05 20:54:49 +04:00
struct {
u8 sha [ ETH_ALEN ] ; /* ARP source hardware address. */
u8 tha [ ETH_ALEN ] ; /* ARP target hardware address. */
} arp ;
2011-10-26 06:26:31 +04:00
} ipv4 ;
struct {
struct {
struct in6_addr src ; /* IPv6 source address. */
struct in6_addr dst ; /* IPv6 destination address. */
} addr ;
__be32 label ; /* IPv6 flow label. */
struct {
struct in6_addr target ; /* ND target address. */
u8 sll [ ETH_ALEN ] ; /* ND source link layer address. */
u8 tll [ ETH_ALEN ] ; /* ND target link layer address. */
} nd ;
} ipv6 ;
} ;
2015-08-26 21:31:48 +03:00
struct {
/* Connection tracking fields. */
u16 zone ;
2015-08-26 21:31:49 +03:00
u32 mark ;
2015-08-26 21:31:48 +03:00
u8 state ;
2015-10-02 01:00:37 +03:00
struct ovs_key_ct_labels labels ;
2015-08-26 21:31:48 +03:00
} ct ;
2013-09-05 23:17:05 +04:00
} __aligned ( BITS_PER_LONG / 8 ) ; /* Ensure that we can do comparisons as longs. */
2011-10-26 06:26:31 +04:00
2013-10-04 05:16:47 +04:00
struct sw_flow_key_range {
2013-11-25 22:41:28 +04:00
unsigned short int start ;
unsigned short int end ;
2013-10-04 05:16:47 +04:00
} ;
struct sw_flow_mask {
int ref_count ;
struct rcu_head rcu ;
struct list_head list ;
struct sw_flow_key_range range ;
struct sw_flow_key key ;
} ;
struct sw_flow_match {
struct sw_flow_key * key ;
struct sw_flow_key_range range ;
struct sw_flow_mask * mask ;
} ;
2015-01-22 03:42:52 +03:00
# define MAX_UFID_LENGTH 16 /* 128 bits */
struct sw_flow_id {
u32 ufid_len ;
union {
u32 ufid [ MAX_UFID_LENGTH / 4 ] ;
struct sw_flow_key * unmasked_key ;
} ;
} ;
2013-10-04 05:16:47 +04:00
struct sw_flow_actions {
struct rcu_head rcu ;
2015-08-26 21:31:44 +03:00
size_t orig_len ; /* From flow_cmd_new netlink actions size */
2013-10-04 05:16:47 +04:00
u32 actions_len ;
struct nlattr actions [ ] ;
} ;
2013-10-30 04:22:21 +04:00
struct flow_stats {
u64 packet_count ; /* Number of packets matched. */
u64 byte_count ; /* Number of bytes matched. */
unsigned long used ; /* Last used time (in jiffies). */
spinlock_t lock ; /* Lock for atomic stats update. */
__be16 tcp_flags ; /* Union of seen TCP flags. */
} ;
2011-10-26 06:26:31 +04:00
struct sw_flow {
struct rcu_head rcu ;
2015-01-22 03:42:52 +03:00
struct {
struct hlist_node node [ 2 ] ;
u32 hash ;
} flow_table , ufid_table ;
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-27 23:42:54 +04:00
int stats_last_writer ; /* NUMA-node id of the last writer on
* ' stats [ 0 ] ' .
*/
2011-10-26 06:26:31 +04:00
struct sw_flow_key key ;
2015-01-22 03:42:52 +03:00
struct sw_flow_id id ;
2013-08-08 07:01:00 +04:00
struct sw_flow_mask * mask ;
2011-10-26 06:26:31 +04:00
struct sw_flow_actions __rcu * sf_acts ;
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-27 23:42:54 +04:00
struct flow_stats __rcu * stats [ ] ; /* One for each NUMA node. First one
* is allocated at flow creation time ,
* the rest are allocated on demand
* while holding the ' stats [ 0 ] . lock ' .
*/
2011-10-26 06:26:31 +04:00
} ;
struct arp_eth_header {
__be16 ar_hrd ; /* format of hardware address */
__be16 ar_pro ; /* format of protocol address */
unsigned char ar_hln ; /* length of hardware address */
unsigned char ar_pln ; /* length of protocol address */
__be16 ar_op ; /* ARP opcode (command) */
/* Ethernet+IPv4 specific members. */
unsigned char ar_sha [ ETH_ALEN ] ; /* sender hardware address */
unsigned char ar_sip [ 4 ] ; /* sender IP address */
unsigned char ar_tha [ ETH_ALEN ] ; /* target hardware address */
unsigned char ar_tip [ 4 ] ; /* target IP address */
} __packed ;
2015-01-22 03:42:52 +03:00
static inline bool ovs_identifier_is_ufid ( const struct sw_flow_id * sfid )
{
return sfid - > ufid_len ;
}
static inline bool ovs_identifier_is_key ( const struct sw_flow_id * sfid )
{
return ! ovs_identifier_is_ufid ( sfid ) ;
}
2014-05-07 03:48:38 +04:00
void ovs_flow_stats_update ( struct sw_flow * , __be16 tcp_flags ,
2014-11-06 17:58:52 +03:00
const struct sk_buff * ) ;
2014-05-06 01:17:28 +04:00
void ovs_flow_stats_get ( const struct sw_flow * , struct ovs_flow_stats * ,
2013-10-30 04:22:21 +04:00
unsigned long * used , __be16 * tcp_flags ) ;
2014-05-06 01:17:28 +04:00
void ovs_flow_stats_clear ( struct sw_flow * ) ;
2011-10-26 06:26:31 +04:00
u64 ovs_flow_used_time ( unsigned long flow_jiffies ) ;
2014-09-16 06:37:25 +04:00
int ovs_flow_key_update ( struct sk_buff * skb , struct sw_flow_key * key ) ;
2015-07-21 11:43:54 +03:00
int ovs_flow_key_extract ( const struct ip_tunnel_info * tun_info ,
2014-11-06 17:58:52 +03:00
struct sk_buff * skb ,
2014-10-04 02:35:31 +04:00
struct sw_flow_key * key ) ;
2014-09-16 06:20:31 +04:00
/* Extract key from packet coming from userspace. */
2015-08-26 21:31:52 +03:00
int ovs_flow_key_extract_userspace ( struct net * net , const struct nlattr * attr ,
2014-09-16 06:20:31 +04:00
struct sk_buff * skb ,
2014-11-06 18:03:05 +03:00
struct sw_flow_key * key , bool log ) ;
2013-08-08 07:01:00 +04:00
2011-10-26 06:26:31 +04:00
# endif /* flow.h */