Merge branch 'selftests-forwarding-Add-new-test-cases'
Ido Schimmel says: ==================== selftests: forwarding: Add new test cases This patchset mainly adds new forwarding test cases and performs small changes in existing infrastructure. Patches #1-#3 add new test cases for multicast RPF check, PCP and VLAN matching using flower and tc VLAN modify action. The rest of the patches are from Petr who says: In patches #4 and #5, devlink_lib.sh is fixed to first not cause double inclusion of lib.sh, and then to deduce the device name in a simpler way. In patch #6, helpers for dealing with shared buffer configuration are added to devlink_lib.sh. In patch #7, MC-awareness test is fixed to configure shared buffers explicitly. In patch #8, several helpers are extracted from the MC-awareness test and put into a new mlxsw-specific library, qos_lib.sh. In patch #9, a new test is added which checks configuration of strictly-prioritized streams. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
commit
113e59d09f
311
tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh
Executable file
311
tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh
Executable file
@ -0,0 +1,311 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# A test for strict prioritization of traffic in the switch. Run two streams of
|
||||
# traffic, each through a different ingress port, one tagged with PCP of 1, the
|
||||
# other with PCP of 2. Both streams converge at one egress port, where they are
|
||||
# assigned TC of, respectively, 1 and 2, with strict priority configured between
|
||||
# them. In H3, we expect to see (almost) exclusively the high-priority traffic.
|
||||
#
|
||||
# Please see qos_mc_aware.sh for an explanation of why we use mausezahn and
|
||||
# counters instead of just running iperf3.
|
||||
#
|
||||
# +---------------------------+ +-----------------------------+
|
||||
# | H1 | | H2 |
|
||||
# | $h1.111 + | | + $h2.222 |
|
||||
# | 192.0.2.33/28 | | | | 192.0.2.65/28 |
|
||||
# | e-qos-map 0:1 | | | | e-qos-map 0:2 |
|
||||
# | | | | | |
|
||||
# | $h1 + | | + $h2 |
|
||||
# +-----------------|---------+ +---------|-------------------+
|
||||
# | |
|
||||
# +-----------------|-------------------------------------|-------------------+
|
||||
# | $swp1 + + $swp2 |
|
||||
# | >1Gbps | | >1Gbps |
|
||||
# | +---------------|-----------+ +----------|----------------+ |
|
||||
# | | $swp1.111 + | | + $swp2.222 | |
|
||||
# | | BR111 | SW | BR222 | |
|
||||
# | | $swp3.111 + | | + $swp3.222 | |
|
||||
# | +---------------|-----------+ +----------|----------------+ |
|
||||
# | \_____________________________________/ |
|
||||
# | | |
|
||||
# | + $swp3 |
|
||||
# | | 1Gbps bottleneck |
|
||||
# | | ETS: (up n->tc n for n in 0..7) |
|
||||
# | | strict priority |
|
||||
# +------------------------------------|--------------------------------------+
|
||||
# |
|
||||
# +--------------------|--------------------+
|
||||
# | + $h3 H3 |
|
||||
# | / \ |
|
||||
# | / \ |
|
||||
# | $h3.111 + + $h3.222 |
|
||||
# | 192.0.2.34/28 192.0.2.66/28 |
|
||||
# +-----------------------------------------+
|
||||
|
||||
ALL_TESTS="
|
||||
ping_ipv4
|
||||
test_ets_strict
|
||||
"
|
||||
|
||||
lib_dir=$(dirname $0)/../../../net/forwarding
|
||||
|
||||
NUM_NETIFS=6
|
||||
source $lib_dir/lib.sh
|
||||
source $lib_dir/devlink_lib.sh
|
||||
source qos_lib.sh
|
||||
|
||||
h1_create()
|
||||
{
|
||||
simple_if_init $h1
|
||||
mtu_set $h1 10000
|
||||
|
||||
vlan_create $h1 111 v$h1 192.0.2.33/28
|
||||
ip link set dev $h1.111 type vlan egress-qos-map 0:1
|
||||
}
|
||||
|
||||
h1_destroy()
|
||||
{
|
||||
vlan_destroy $h1 111
|
||||
|
||||
mtu_restore $h1
|
||||
simple_if_fini $h1
|
||||
}
|
||||
|
||||
h2_create()
|
||||
{
|
||||
simple_if_init $h2
|
||||
mtu_set $h2 10000
|
||||
|
||||
vlan_create $h2 222 v$h2 192.0.2.65/28
|
||||
ip link set dev $h2.222 type vlan egress-qos-map 0:2
|
||||
}
|
||||
|
||||
h2_destroy()
|
||||
{
|
||||
vlan_destroy $h2 222
|
||||
|
||||
mtu_restore $h2
|
||||
simple_if_fini $h2
|
||||
}
|
||||
|
||||
h3_create()
|
||||
{
|
||||
simple_if_init $h3
|
||||
mtu_set $h3 10000
|
||||
|
||||
vlan_create $h3 111 v$h3 192.0.2.34/28
|
||||
vlan_create $h3 222 v$h3 192.0.2.66/28
|
||||
}
|
||||
|
||||
h3_destroy()
|
||||
{
|
||||
vlan_destroy $h3 222
|
||||
vlan_destroy $h3 111
|
||||
|
||||
mtu_restore $h3
|
||||
simple_if_fini $h3
|
||||
}
|
||||
|
||||
switch_create()
|
||||
{
|
||||
ip link set dev $swp1 up
|
||||
mtu_set $swp1 10000
|
||||
|
||||
ip link set dev $swp2 up
|
||||
mtu_set $swp2 10000
|
||||
|
||||
# prio n -> TC n, strict scheduling
|
||||
lldptool -T -i $swp3 -V ETS-CFG up2tc=0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7
|
||||
lldptool -T -i $swp3 -V ETS-CFG tsa=$(
|
||||
)"0:strict,"$(
|
||||
)"1:strict,"$(
|
||||
)"2:strict,"$(
|
||||
)"3:strict,"$(
|
||||
)"4:strict,"$(
|
||||
)"5:strict,"$(
|
||||
)"6:strict,"$(
|
||||
)"7:strict"
|
||||
sleep 1
|
||||
|
||||
ip link set dev $swp3 up
|
||||
mtu_set $swp3 10000
|
||||
ethtool -s $swp3 speed 1000 autoneg off
|
||||
|
||||
vlan_create $swp1 111
|
||||
vlan_create $swp2 222
|
||||
vlan_create $swp3 111
|
||||
vlan_create $swp3 222
|
||||
|
||||
ip link add name br111 up type bridge vlan_filtering 0
|
||||
ip link set dev $swp1.111 master br111
|
||||
ip link set dev $swp3.111 master br111
|
||||
|
||||
ip link add name br222 up type bridge vlan_filtering 0
|
||||
ip link set dev $swp2.222 master br222
|
||||
ip link set dev $swp3.222 master br222
|
||||
|
||||
# Make sure that ingress quotas are smaller than egress so that there is
|
||||
# room for both streams of traffic to be admitted to shared buffer.
|
||||
devlink_pool_size_thtype_set 0 dynamic 10000000
|
||||
devlink_pool_size_thtype_set 4 dynamic 10000000
|
||||
|
||||
devlink_port_pool_th_set $swp1 0 6
|
||||
devlink_tc_bind_pool_th_set $swp1 1 ingress 0 6
|
||||
|
||||
devlink_port_pool_th_set $swp2 0 6
|
||||
devlink_tc_bind_pool_th_set $swp2 2 ingress 0 6
|
||||
|
||||
devlink_tc_bind_pool_th_set $swp3 1 egress 4 7
|
||||
devlink_tc_bind_pool_th_set $swp3 2 egress 4 7
|
||||
devlink_port_pool_th_set $swp3 4 7
|
||||
}
|
||||
|
||||
switch_destroy()
|
||||
{
|
||||
devlink_port_pool_th_restore $swp3 4
|
||||
devlink_tc_bind_pool_th_restore $swp3 2 egress
|
||||
devlink_tc_bind_pool_th_restore $swp3 1 egress
|
||||
|
||||
devlink_tc_bind_pool_th_restore $swp2 2 ingress
|
||||
devlink_port_pool_th_restore $swp2 0
|
||||
|
||||
devlink_tc_bind_pool_th_restore $swp1 1 ingress
|
||||
devlink_port_pool_th_restore $swp1 0
|
||||
|
||||
devlink_pool_size_thtype_restore 4
|
||||
devlink_pool_size_thtype_restore 0
|
||||
|
||||
ip link del dev br222
|
||||
ip link del dev br111
|
||||
|
||||
vlan_destroy $swp3 222
|
||||
vlan_destroy $swp3 111
|
||||
vlan_destroy $swp2 222
|
||||
vlan_destroy $swp1 111
|
||||
|
||||
ethtool -s $swp3 autoneg on
|
||||
mtu_restore $swp3
|
||||
ip link set dev $swp3 down
|
||||
lldptool -T -i $swp3 -V ETS-CFG up2tc=0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0
|
||||
|
||||
mtu_restore $swp2
|
||||
ip link set dev $swp2 down
|
||||
|
||||
mtu_restore $swp1
|
||||
ip link set dev $swp1 down
|
||||
}
|
||||
|
||||
setup_prepare()
|
||||
{
|
||||
h1=${NETIFS[p1]}
|
||||
swp1=${NETIFS[p2]}
|
||||
|
||||
swp2=${NETIFS[p3]}
|
||||
h2=${NETIFS[p4]}
|
||||
|
||||
swp3=${NETIFS[p5]}
|
||||
h3=${NETIFS[p6]}
|
||||
|
||||
h3mac=$(mac_get $h3)
|
||||
|
||||
vrf_prepare
|
||||
|
||||
h1_create
|
||||
h2_create
|
||||
h3_create
|
||||
switch_create
|
||||
}
|
||||
|
||||
cleanup()
|
||||
{
|
||||
pre_cleanup
|
||||
|
||||
switch_destroy
|
||||
h3_destroy
|
||||
h2_destroy
|
||||
h1_destroy
|
||||
|
||||
vrf_cleanup
|
||||
}
|
||||
|
||||
ping_ipv4()
|
||||
{
|
||||
ping_test $h1 192.0.2.34 " from H1"
|
||||
ping_test $h2 192.0.2.66 " from H2"
|
||||
}
|
||||
|
||||
rel()
|
||||
{
|
||||
local old=$1; shift
|
||||
local new=$1; shift
|
||||
|
||||
bc <<< "
|
||||
scale=2
|
||||
ret = 100 * $new / $old
|
||||
if (ret > 0) { ret } else { 0 }
|
||||
"
|
||||
}
|
||||
|
||||
test_ets_strict()
|
||||
{
|
||||
RET=0
|
||||
|
||||
# Run high-prio traffic on its own.
|
||||
start_traffic $h2.222 192.0.2.65 192.0.2.66 $h3mac
|
||||
local -a rate_2
|
||||
rate_2=($(measure_rate $swp2 $h3 rx_octets_prio_2 "prio 2"))
|
||||
check_err $? "Could not get high enough prio-2 ingress rate"
|
||||
local rate_2_in=${rate_2[0]}
|
||||
local rate_2_eg=${rate_2[1]}
|
||||
stop_traffic # $h2.222
|
||||
|
||||
# Start low-prio stream.
|
||||
start_traffic $h1.111 192.0.2.33 192.0.2.34 $h3mac
|
||||
|
||||
local -a rate_1
|
||||
rate_1=($(measure_rate $swp1 $h3 rx_octets_prio_1 "prio 1"))
|
||||
check_err $? "Could not get high enough prio-1 ingress rate"
|
||||
local rate_1_in=${rate_1[0]}
|
||||
local rate_1_eg=${rate_1[1]}
|
||||
|
||||
# High-prio and low-prio on their own should have about the same
|
||||
# throughput.
|
||||
local rel21=$(rel $rate_1_eg $rate_2_eg)
|
||||
check_err $(bc <<< "$rel21 < 95")
|
||||
check_err $(bc <<< "$rel21 > 105")
|
||||
|
||||
# Start the high-prio stream--now both streams run.
|
||||
start_traffic $h2.222 192.0.2.65 192.0.2.66 $h3mac
|
||||
rate_3=($(measure_rate $swp2 $h3 rx_octets_prio_2 "prio 2 w/ 1"))
|
||||
check_err $? "Could not get high enough prio-2 ingress rate with prio-1"
|
||||
local rate_3_in=${rate_3[0]}
|
||||
local rate_3_eg=${rate_3[1]}
|
||||
stop_traffic # $h2.222
|
||||
|
||||
stop_traffic # $h1.111
|
||||
|
||||
# High-prio should have about the same throughput whether or not
|
||||
# low-prio is in the system.
|
||||
local rel32=$(rel $rate_2_eg $rate_3_eg)
|
||||
check_err $(bc <<< "$rel32 < 95")
|
||||
|
||||
log_test "strict priority"
|
||||
echo "Ingress to switch:"
|
||||
echo " p1 in rate $(humanize $rate_1_in)"
|
||||
echo " p2 in rate $(humanize $rate_2_in)"
|
||||
echo " p2 in rate w/ p1 $(humanize $rate_3_in)"
|
||||
echo "Egress from switch:"
|
||||
echo " p1 eg rate $(humanize $rate_1_eg)"
|
||||
echo " p2 eg rate $(humanize $rate_2_eg) ($rel21% of p1)"
|
||||
echo " p2 eg rate w/ p1 $(humanize $rate_3_eg) ($rel32% of p2)"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
setup_prepare
|
||||
setup_wait
|
||||
|
||||
tests_run
|
||||
|
||||
exit $EXIT_STATUS
|
||||
98
tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh
Normal file
98
tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh
Normal file
@ -0,0 +1,98 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
humanize()
|
||||
{
|
||||
local speed=$1; shift
|
||||
|
||||
for unit in bps Kbps Mbps Gbps; do
|
||||
if (($(echo "$speed < 1024" | bc))); then
|
||||
break
|
||||
fi
|
||||
|
||||
speed=$(echo "scale=1; $speed / 1024" | bc)
|
||||
done
|
||||
|
||||
echo "$speed${unit}"
|
||||
}
|
||||
|
||||
rate()
|
||||
{
|
||||
local t0=$1; shift
|
||||
local t1=$1; shift
|
||||
local interval=$1; shift
|
||||
|
||||
echo $((8 * (t1 - t0) / interval))
|
||||
}
|
||||
|
||||
start_traffic()
|
||||
{
|
||||
local h_in=$1; shift # Where the traffic egresses the host
|
||||
local sip=$1; shift
|
||||
local dip=$1; shift
|
||||
local dmac=$1; shift
|
||||
|
||||
$MZ $h_in -p 8000 -A $sip -B $dip -c 0 \
|
||||
-a own -b $dmac -t udp -q &
|
||||
sleep 1
|
||||
}
|
||||
|
||||
stop_traffic()
|
||||
{
|
||||
# Suppress noise from killing mausezahn.
|
||||
{ kill %% && wait %%; } 2>/dev/null
|
||||
}
|
||||
|
||||
check_rate()
|
||||
{
|
||||
local rate=$1; shift
|
||||
local min=$1; shift
|
||||
local what=$1; shift
|
||||
|
||||
if ((rate > min)); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "$what $(humanize $ir) < $(humanize $min)" > /dev/stderr
|
||||
return 1
|
||||
}
|
||||
|
||||
measure_rate()
|
||||
{
|
||||
local sw_in=$1; shift # Where the traffic ingresses the switch
|
||||
local host_in=$1; shift # Where it ingresses another host
|
||||
local counter=$1; shift # Counter to use for measurement
|
||||
local what=$1; shift
|
||||
|
||||
local interval=10
|
||||
local i
|
||||
local ret=0
|
||||
|
||||
# Dips in performance might cause momentary ingress rate to drop below
|
||||
# 1Gbps. That wouldn't saturate egress and MC would thus get through,
|
||||
# seemingly winning bandwidth on account of UC. Demand at least 2Gbps
|
||||
# average ingress rate to somewhat mitigate this.
|
||||
local min_ingress=2147483648
|
||||
|
||||
for i in {5..0}; do
|
||||
local t0=$(ethtool_stats_get $host_in $counter)
|
||||
local u0=$(ethtool_stats_get $sw_in $counter)
|
||||
sleep $interval
|
||||
local t1=$(ethtool_stats_get $host_in $counter)
|
||||
local u1=$(ethtool_stats_get $sw_in $counter)
|
||||
|
||||
local ir=$(rate $u0 $u1 $interval)
|
||||
local er=$(rate $t0 $t1 $interval)
|
||||
|
||||
if check_rate $ir $min_ingress "$what ingress rate"; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Fail the test if we can't get the throughput.
|
||||
if ((i == 0)); then
|
||||
ret=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo $ir $er
|
||||
return $ret
|
||||
}
|
||||
@ -67,6 +67,8 @@ lib_dir=$(dirname $0)/../../../net/forwarding
|
||||
|
||||
NUM_NETIFS=6
|
||||
source $lib_dir/lib.sh
|
||||
source $lib_dir/devlink_lib.sh
|
||||
source qos_lib.sh
|
||||
|
||||
h1_create()
|
||||
{
|
||||
@ -140,10 +142,28 @@ switch_create()
|
||||
ip link set dev br111 up
|
||||
ip link set dev $swp2.111 master br111
|
||||
ip link set dev $swp3.111 master br111
|
||||
|
||||
# Make sure that ingress quotas are smaller than egress so that there is
|
||||
# room for both streams of traffic to be admitted to shared buffer.
|
||||
devlink_port_pool_th_set $swp1 0 5
|
||||
devlink_tc_bind_pool_th_set $swp1 0 ingress 0 5
|
||||
|
||||
devlink_port_pool_th_set $swp2 0 5
|
||||
devlink_tc_bind_pool_th_set $swp2 1 ingress 0 5
|
||||
|
||||
devlink_port_pool_th_set $swp3 4 12
|
||||
}
|
||||
|
||||
switch_destroy()
|
||||
{
|
||||
devlink_port_pool_th_restore $swp3 4
|
||||
|
||||
devlink_tc_bind_pool_th_restore $swp2 1 ingress
|
||||
devlink_port_pool_th_restore $swp2 0
|
||||
|
||||
devlink_tc_bind_pool_th_restore $swp1 0 ingress
|
||||
devlink_port_pool_th_restore $swp1 0
|
||||
|
||||
ip link del dev br111
|
||||
ip link del dev br1
|
||||
|
||||
@ -201,107 +221,28 @@ ping_ipv4()
|
||||
ping_test $h2 192.0.2.130
|
||||
}
|
||||
|
||||
humanize()
|
||||
{
|
||||
local speed=$1; shift
|
||||
|
||||
for unit in bps Kbps Mbps Gbps; do
|
||||
if (($(echo "$speed < 1024" | bc))); then
|
||||
break
|
||||
fi
|
||||
|
||||
speed=$(echo "scale=1; $speed / 1024" | bc)
|
||||
done
|
||||
|
||||
echo "$speed${unit}"
|
||||
}
|
||||
|
||||
rate()
|
||||
{
|
||||
local t0=$1; shift
|
||||
local t1=$1; shift
|
||||
local interval=$1; shift
|
||||
|
||||
echo $((8 * (t1 - t0) / interval))
|
||||
}
|
||||
|
||||
check_rate()
|
||||
{
|
||||
local rate=$1; shift
|
||||
local min=$1; shift
|
||||
local what=$1; shift
|
||||
|
||||
if ((rate > min)); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "$what $(humanize $ir) < $(humanize $min_ingress)" > /dev/stderr
|
||||
return 1
|
||||
}
|
||||
|
||||
measure_uc_rate()
|
||||
{
|
||||
local what=$1; shift
|
||||
|
||||
local interval=10
|
||||
local i
|
||||
local ret=0
|
||||
|
||||
# Dips in performance might cause momentary ingress rate to drop below
|
||||
# 1Gbps. That wouldn't saturate egress and MC would thus get through,
|
||||
# seemingly winning bandwidth on account of UC. Demand at least 2Gbps
|
||||
# average ingress rate to somewhat mitigate this.
|
||||
local min_ingress=2147483648
|
||||
|
||||
$MZ $h2.111 -p 8000 -A 192.0.2.129 -B 192.0.2.130 -c 0 \
|
||||
-a own -b $h3mac -t udp -q &
|
||||
sleep 1
|
||||
|
||||
for i in {5..0}; do
|
||||
local t0=$(ethtool_stats_get $h3 rx_octets_prio_1)
|
||||
local u0=$(ethtool_stats_get $swp2 rx_octets_prio_1)
|
||||
sleep $interval
|
||||
local t1=$(ethtool_stats_get $h3 rx_octets_prio_1)
|
||||
local u1=$(ethtool_stats_get $swp2 rx_octets_prio_1)
|
||||
|
||||
local ir=$(rate $u0 $u1 $interval)
|
||||
local er=$(rate $t0 $t1 $interval)
|
||||
|
||||
if check_rate $ir $min_ingress "$what ingress rate"; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Fail the test if we can't get the throughput.
|
||||
if ((i == 0)); then
|
||||
ret=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Suppress noise from killing mausezahn.
|
||||
{ kill %% && wait; } 2>/dev/null
|
||||
|
||||
echo $ir $er
|
||||
exit $ret
|
||||
}
|
||||
|
||||
test_mc_aware()
|
||||
{
|
||||
RET=0
|
||||
|
||||
local -a uc_rate
|
||||
uc_rate=($(measure_uc_rate "UC-only"))
|
||||
start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac
|
||||
uc_rate=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC-only"))
|
||||
check_err $? "Could not get high enough UC-only ingress rate"
|
||||
stop_traffic
|
||||
local ucth1=${uc_rate[1]}
|
||||
|
||||
$MZ $h1 -p 8000 -c 0 -a own -b bc -t udp -q &
|
||||
start_traffic $h1 own bc bc
|
||||
|
||||
local d0=$(date +%s)
|
||||
local t0=$(ethtool_stats_get $h3 rx_octets_prio_0)
|
||||
local u0=$(ethtool_stats_get $swp1 rx_octets_prio_0)
|
||||
|
||||
local -a uc_rate_2
|
||||
uc_rate_2=($(measure_uc_rate "UC+MC"))
|
||||
start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac
|
||||
uc_rate_2=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC+MC"))
|
||||
check_err $? "Could not get high enough UC+MC ingress rate"
|
||||
stop_traffic
|
||||
local ucth2=${uc_rate_2[1]}
|
||||
|
||||
local d1=$(date +%s)
|
||||
@ -319,8 +260,7 @@ test_mc_aware()
|
||||
local mc_ir=$(rate $u0 $u1 $interval)
|
||||
local mc_er=$(rate $t0 $t1 $interval)
|
||||
|
||||
# Suppress noise from killing mausezahn.
|
||||
{ kill %% && wait; } 2>/dev/null
|
||||
stop_traffic
|
||||
|
||||
log_test "UC performace under MC overload"
|
||||
|
||||
@ -344,8 +284,7 @@ test_uc_aware()
|
||||
{
|
||||
RET=0
|
||||
|
||||
$MZ $h2.111 -p 8000 -A 192.0.2.129 -B 192.0.2.130 -c 0 \
|
||||
-a own -b $h3mac -t udp -q &
|
||||
start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac
|
||||
|
||||
local d0=$(date +%s)
|
||||
local t0=$(ethtool_stats_get $h3 rx_octets_prio_1)
|
||||
@ -375,8 +314,7 @@ test_uc_aware()
|
||||
((attempts == passes))
|
||||
check_err $?
|
||||
|
||||
# Suppress noise from killing mausezahn.
|
||||
{ kill %% && wait; } 2>/dev/null
|
||||
stop_traffic
|
||||
|
||||
log_test "MC performace under UC overload"
|
||||
echo " ingress UC throughput $(humanize ${uc_ir})"
|
||||
|
||||
@ -12,6 +12,7 @@ ALL_TESTS="single_mask_test identical_filters_test two_masks_test \
|
||||
delta_two_masks_one_key_test delta_simple_rehash_test \
|
||||
bloom_simple_test bloom_complex_test bloom_delta_test"
|
||||
NUM_NETIFS=2
|
||||
source $lib_dir/lib.sh
|
||||
source $lib_dir/tc_common.sh
|
||||
source $lib_dir/devlink_lib.sh
|
||||
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
lib_dir=$(dirname $0)/../../../../net/forwarding
|
||||
|
||||
NUM_NETIFS=1
|
||||
source $lib_dir/lib.sh
|
||||
source devlink_lib_spectrum.sh
|
||||
|
||||
setup_prepare()
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
lib_dir=$(dirname $0)/../../../../net/forwarding
|
||||
|
||||
NUM_NETIFS=6
|
||||
source ../../../../net/forwarding/tc_common.sh
|
||||
source $lib_dir/lib.sh
|
||||
source $lib_dir/tc_common.sh
|
||||
source devlink_lib_spectrum.sh
|
||||
|
||||
current_test=""
|
||||
|
||||
@ -1,22 +1,11 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
##############################################################################
|
||||
# Source library
|
||||
|
||||
relative_path="${BASH_SOURCE%/*}"
|
||||
if [[ "$relative_path" == "${BASH_SOURCE}" ]]; then
|
||||
relative_path="."
|
||||
fi
|
||||
|
||||
source "$relative_path/lib.sh"
|
||||
|
||||
##############################################################################
|
||||
# Defines
|
||||
|
||||
DEVLINK_DEV=$(devlink port show | grep "${NETIFS[p1]}" | \
|
||||
grep -v "${NETIFS[p1]}[0-9]" | cut -d" " -f1 | \
|
||||
rev | cut -d"/" -f2- | rev)
|
||||
DEVLINK_DEV=$(devlink port show "${NETIFS[p1]}" -j \
|
||||
| jq -r '.port | keys[]' | cut -d/ -f-2)
|
||||
if [ -z "$DEVLINK_DEV" ]; then
|
||||
echo "SKIP: ${NETIFS[p1]} has no devlink device registered for it"
|
||||
exit 1
|
||||
@ -106,3 +95,98 @@ devlink_reload()
|
||||
grep -c "size_new")
|
||||
check_err $still_pending "Failed reload - There are still unset sizes"
|
||||
}
|
||||
|
||||
declare -A DEVLINK_ORIG
|
||||
|
||||
devlink_port_pool_threshold()
|
||||
{
|
||||
local port=$1; shift
|
||||
local pool=$1; shift
|
||||
|
||||
devlink sb port pool show $port pool $pool -j \
|
||||
| jq '.port_pool."'"$port"'"[].threshold'
|
||||
}
|
||||
|
||||
devlink_port_pool_th_set()
|
||||
{
|
||||
local port=$1; shift
|
||||
local pool=$1; shift
|
||||
local th=$1; shift
|
||||
local key="port_pool($port,$pool).threshold"
|
||||
|
||||
DEVLINK_ORIG[$key]=$(devlink_port_pool_threshold $port $pool)
|
||||
devlink sb port pool set $port pool $pool th $th
|
||||
}
|
||||
|
||||
devlink_port_pool_th_restore()
|
||||
{
|
||||
local port=$1; shift
|
||||
local pool=$1; shift
|
||||
local key="port_pool($port,$pool).threshold"
|
||||
|
||||
devlink sb port pool set $port pool $pool th ${DEVLINK_ORIG[$key]}
|
||||
}
|
||||
|
||||
devlink_pool_size_thtype()
|
||||
{
|
||||
local pool=$1; shift
|
||||
|
||||
devlink sb pool show "$DEVLINK_DEV" pool $pool -j \
|
||||
| jq -r '.pool[][] | (.size, .thtype)'
|
||||
}
|
||||
|
||||
devlink_pool_size_thtype_set()
|
||||
{
|
||||
local pool=$1; shift
|
||||
local thtype=$1; shift
|
||||
local size=$1; shift
|
||||
local key="pool($pool).size_thtype"
|
||||
|
||||
DEVLINK_ORIG[$key]=$(devlink_pool_size_thtype $pool)
|
||||
devlink sb pool set "$DEVLINK_DEV" pool $pool size $size thtype $thtype
|
||||
}
|
||||
|
||||
devlink_pool_size_thtype_restore()
|
||||
{
|
||||
local pool=$1; shift
|
||||
local key="pool($pool).size_thtype"
|
||||
local -a orig=(${DEVLINK_ORIG[$key]})
|
||||
|
||||
devlink sb pool set "$DEVLINK_DEV" pool $pool \
|
||||
size ${orig[0]} thtype ${orig[1]}
|
||||
}
|
||||
|
||||
devlink_tc_bind_pool_th()
|
||||
{
|
||||
local port=$1; shift
|
||||
local tc=$1; shift
|
||||
local dir=$1; shift
|
||||
|
||||
devlink sb tc bind show $port tc $tc type $dir -j \
|
||||
| jq -r '.tc_bind[][] | (.pool, .threshold)'
|
||||
}
|
||||
|
||||
devlink_tc_bind_pool_th_set()
|
||||
{
|
||||
local port=$1; shift
|
||||
local tc=$1; shift
|
||||
local dir=$1; shift
|
||||
local pool=$1; shift
|
||||
local th=$1; shift
|
||||
local key="tc_bind($port,$dir,$tc).pool_th"
|
||||
|
||||
DEVLINK_ORIG[$key]=$(devlink_tc_bind_pool_th $port $tc $dir)
|
||||
devlink sb tc bind set $port tc $tc type $dir pool $pool th $th
|
||||
}
|
||||
|
||||
devlink_tc_bind_pool_th_restore()
|
||||
{
|
||||
local port=$1; shift
|
||||
local tc=$1; shift
|
||||
local dir=$1; shift
|
||||
local key="tc_bind($port,$dir,$tc).pool_th"
|
||||
local -a orig=(${DEVLINK_ORIG[$key]})
|
||||
|
||||
devlink sb tc bind set $port tc $tc type $dir \
|
||||
pool ${orig[0]} th ${orig[1]}
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@
|
||||
# +------------------+ +------------------+
|
||||
#
|
||||
|
||||
ALL_TESTS="mcast_v4 mcast_v6"
|
||||
ALL_TESTS="mcast_v4 mcast_v6 rpf_v4 rpf_v6"
|
||||
NUM_NETIFS=6
|
||||
source lib.sh
|
||||
source tc_common.sh
|
||||
@ -46,10 +46,14 @@ h1_create()
|
||||
|
||||
ip route add 2001:db8:2::/64 vrf v$h1 nexthop via 2001:db8:1::1
|
||||
ip route add 2001:db8:3::/64 vrf v$h1 nexthop via 2001:db8:1::1
|
||||
|
||||
tc qdisc add dev $h1 ingress
|
||||
}
|
||||
|
||||
h1_destroy()
|
||||
{
|
||||
tc qdisc del dev $h1 ingress
|
||||
|
||||
ip route del 2001:db8:3::/64 vrf v$h1
|
||||
ip route del 2001:db8:2::/64 vrf v$h1
|
||||
|
||||
@ -124,10 +128,14 @@ router_create()
|
||||
ip address add 2001:db8:1::1/64 dev $rp1
|
||||
ip address add 2001:db8:2::1/64 dev $rp2
|
||||
ip address add 2001:db8:3::1/64 dev $rp3
|
||||
|
||||
tc qdisc add dev $rp3 ingress
|
||||
}
|
||||
|
||||
router_destroy()
|
||||
{
|
||||
tc qdisc del dev $rp3 ingress
|
||||
|
||||
ip address del 2001:db8:3::1/64 dev $rp3
|
||||
ip address del 2001:db8:2::1/64 dev $rp2
|
||||
ip address del 2001:db8:1::1/64 dev $rp1
|
||||
@ -301,6 +309,103 @@ mcast_v6()
|
||||
log_test "mcast IPv6"
|
||||
}
|
||||
|
||||
rpf_v4()
|
||||
{
|
||||
# Add a multicast route from first router port to the other two. Send
|
||||
# matching packets and test that both hosts receive them. Then, send
|
||||
# the same packets via the third router port and test that they do not
|
||||
# reach any host due to RPF check. A filter with 'skip_hw' is added to
|
||||
# test that devices capable of multicast routing offload trap those
|
||||
# packets. The filter is essentialy a NOP in other scenarios.
|
||||
|
||||
RET=0
|
||||
|
||||
tc filter add dev $h1 ingress protocol ip pref 1 handle 1 flower \
|
||||
dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $h2 ingress protocol ip pref 1 handle 1 flower \
|
||||
dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $h3 ingress protocol ip pref 1 handle 1 flower \
|
||||
dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $rp3 ingress protocol ip pref 1 handle 1 flower \
|
||||
skip_hw dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action pass
|
||||
|
||||
create_mcast_sg $rp1 198.51.100.2 225.1.2.3 $rp2 $rp3
|
||||
|
||||
$MZ $h1 -c 5 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \
|
||||
-a 00:11:22:33:44:55 -b 01:00:5e:01:02:03 \
|
||||
-A 198.51.100.2 -B 225.1.2.3 -q
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 1 5
|
||||
check_err $? "Multicast not received on first host"
|
||||
tc_check_packets "dev $h3 ingress" 1 5
|
||||
check_err $? "Multicast not received on second host"
|
||||
|
||||
$MZ $h3 -c 5 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \
|
||||
-a 00:11:22:33:44:55 -b 01:00:5e:01:02:03 \
|
||||
-A 198.51.100.2 -B 225.1.2.3 -q
|
||||
|
||||
tc_check_packets "dev $h1 ingress" 1 0
|
||||
check_err $? "Multicast received on first host when should not"
|
||||
tc_check_packets "dev $h2 ingress" 1 5
|
||||
check_err $? "Multicast received on second host when should not"
|
||||
tc_check_packets "dev $rp3 ingress" 1 5
|
||||
check_err $? "Packets not trapped due to RPF check"
|
||||
|
||||
delete_mcast_sg $rp1 198.51.100.2 225.1.2.3 $rp2 $rp3
|
||||
|
||||
tc filter del dev $rp3 ingress protocol ip pref 1 handle 1 flower
|
||||
tc filter del dev $h3 ingress protocol ip pref 1 handle 1 flower
|
||||
tc filter del dev $h2 ingress protocol ip pref 1 handle 1 flower
|
||||
tc filter del dev $h1 ingress protocol ip pref 1 handle 1 flower
|
||||
|
||||
log_test "RPF IPv4"
|
||||
}
|
||||
|
||||
rpf_v6()
|
||||
{
|
||||
RET=0
|
||||
|
||||
tc filter add dev $h1 ingress protocol ipv6 pref 1 handle 1 flower \
|
||||
dst_ip ff0e::3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $h2 ingress protocol ipv6 pref 1 handle 1 flower \
|
||||
dst_ip ff0e::3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $h3 ingress protocol ipv6 pref 1 handle 1 flower \
|
||||
dst_ip ff0e::3 ip_proto udp dst_port 12345 action drop
|
||||
tc filter add dev $rp3 ingress protocol ipv6 pref 1 handle 1 flower \
|
||||
skip_hw dst_ip ff0e::3 ip_proto udp dst_port 12345 action pass
|
||||
|
||||
create_mcast_sg $rp1 2001:db8:1::2 ff0e::3 $rp2 $rp3
|
||||
|
||||
$MZ $h1 -6 -c 5 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \
|
||||
-a 00:11:22:33:44:55 -b 33:33:00:00:00:03 \
|
||||
-A 2001:db8:1::2 -B ff0e::3 -q
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 1 5
|
||||
check_err $? "Multicast not received on first host"
|
||||
tc_check_packets "dev $h3 ingress" 1 5
|
||||
check_err $? "Multicast not received on second host"
|
||||
|
||||
$MZ $h3 -6 -c 5 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \
|
||||
-a 00:11:22:33:44:55 -b 33:33:00:00:00:03 \
|
||||
-A 2001:db8:1::2 -B ff0e::3 -q
|
||||
|
||||
tc_check_packets "dev $h1 ingress" 1 0
|
||||
check_err $? "Multicast received on first host when should not"
|
||||
tc_check_packets "dev $h2 ingress" 1 5
|
||||
check_err $? "Multicast received on second host when should not"
|
||||
tc_check_packets "dev $rp3 ingress" 1 5
|
||||
check_err $? "Packets not trapped due to RPF check"
|
||||
|
||||
delete_mcast_sg $rp1 2001:db8:1::2 ff0e::3 $rp2 $rp3
|
||||
|
||||
tc filter del dev $rp3 ingress protocol ipv6 pref 1 handle 1 flower
|
||||
tc filter del dev $h3 ingress protocol ipv6 pref 1 handle 1 flower
|
||||
tc filter del dev $h2 ingress protocol ipv6 pref 1 handle 1 flower
|
||||
tc filter del dev $h1 ingress protocol ipv6 pref 1 handle 1 flower
|
||||
|
||||
log_test "RPF IPv6"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
setup_prepare
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
ALL_TESTS="match_dst_mac_test match_src_mac_test match_dst_ip_test \
|
||||
match_src_ip_test match_ip_flags_test"
|
||||
match_src_ip_test match_ip_flags_test match_pcp_test match_vlan_test"
|
||||
NUM_NETIFS=2
|
||||
source tc_common.sh
|
||||
source lib.sh
|
||||
@ -219,6 +219,63 @@ match_ip_flags_test()
|
||||
log_test "ip_flags match ($tcflags)"
|
||||
}
|
||||
|
||||
match_pcp_test()
|
||||
{
|
||||
RET=0
|
||||
|
||||
vlan_create $h2 85 v$h2 192.0.2.11/24
|
||||
|
||||
tc filter add dev $h2 ingress protocol 802.1q pref 1 handle 101 \
|
||||
flower vlan_prio 6 $tcflags dst_mac $h2mac action drop
|
||||
tc filter add dev $h2 ingress protocol 802.1q pref 2 handle 102 \
|
||||
flower vlan_prio 7 $tcflags dst_mac $h2mac action drop
|
||||
|
||||
$MZ $h1 -c 1 -p 64 -a $h1mac -b $h2mac -B 192.0.2.11 -Q 7:85 -t ip -q
|
||||
$MZ $h1 -c 1 -p 64 -a $h1mac -b $h2mac -B 192.0.2.11 -Q 0:85 -t ip -q
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 101 0
|
||||
check_err $? "Matched on specified PCP when should not"
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 102 1
|
||||
check_err $? "Did not match on specified PCP"
|
||||
|
||||
tc filter del dev $h2 ingress protocol 802.1q pref 2 handle 102 flower
|
||||
tc filter del dev $h2 ingress protocol 802.1q pref 1 handle 101 flower
|
||||
|
||||
vlan_destroy $h2 85
|
||||
|
||||
log_test "PCP match ($tcflags)"
|
||||
}
|
||||
|
||||
match_vlan_test()
|
||||
{
|
||||
RET=0
|
||||
|
||||
vlan_create $h2 85 v$h2 192.0.2.11/24
|
||||
vlan_create $h2 75 v$h2 192.0.2.10/24
|
||||
|
||||
tc filter add dev $h2 ingress protocol 802.1q pref 1 handle 101 \
|
||||
flower vlan_id 75 $tcflags action drop
|
||||
tc filter add dev $h2 ingress protocol 802.1q pref 2 handle 102 \
|
||||
flower vlan_id 85 $tcflags action drop
|
||||
|
||||
$MZ $h1 -c 1 -p 64 -a $h1mac -b $h2mac -B 192.0.2.11 -Q 0:85 -t ip -q
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 101 0
|
||||
check_err $? "Matched on specified VLAN when should not"
|
||||
|
||||
tc_check_packets "dev $h2 ingress" 102 1
|
||||
check_err $? "Did not match on specified VLAN"
|
||||
|
||||
tc filter del dev $h2 ingress protocol 802.1q pref 2 handle 102 flower
|
||||
tc filter del dev $h2 ingress protocol 802.1q pref 1 handle 101 flower
|
||||
|
||||
vlan_destroy $h2 75
|
||||
vlan_destroy $h2 85
|
||||
|
||||
log_test "VLAN match ($tcflags)"
|
||||
}
|
||||
|
||||
setup_prepare()
|
||||
{
|
||||
h1=${NETIFS[p1]}
|
||||
|
||||
164
tools/testing/selftests/net/forwarding/tc_vlan_modify.sh
Executable file
164
tools/testing/selftests/net/forwarding/tc_vlan_modify.sh
Executable file
@ -0,0 +1,164 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
ALL_TESTS="
|
||||
vlan_modify_ingress
|
||||
vlan_modify_egress
|
||||
"
|
||||
|
||||
NUM_NETIFS=4
|
||||
CHECK_TC="yes"
|
||||
source lib.sh
|
||||
|
||||
h1_create()
|
||||
{
|
||||
simple_if_init $h1 192.0.2.1/28 2001:db8:1::1/64
|
||||
vlan_create $h1 85 v$h1 192.0.2.17/28 2001:db8:2::1/64
|
||||
}
|
||||
|
||||
h1_destroy()
|
||||
{
|
||||
vlan_destroy $h1 85
|
||||
simple_if_fini $h1 192.0.2.1/28 2001:db8:1::1/64
|
||||
}
|
||||
|
||||
h2_create()
|
||||
{
|
||||
simple_if_init $h2 192.0.2.2/28 2001:db8:1::2/64
|
||||
vlan_create $h2 65 v$h2 192.0.2.18/28 2001:db8:2::2/64
|
||||
}
|
||||
|
||||
h2_destroy()
|
||||
{
|
||||
vlan_destroy $h2 65
|
||||
simple_if_fini $h2 192.0.2.2/28 2001:db8:1::2/64
|
||||
}
|
||||
|
||||
switch_create()
|
||||
{
|
||||
ip link add dev br0 type bridge vlan_filtering 1 mcast_snooping 0
|
||||
|
||||
ip link set dev $swp1 master br0
|
||||
ip link set dev $swp2 master br0
|
||||
|
||||
ip link set dev br0 up
|
||||
ip link set dev $swp1 up
|
||||
ip link set dev $swp2 up
|
||||
|
||||
bridge vlan add dev $swp1 vid 85
|
||||
bridge vlan add dev $swp2 vid 65
|
||||
|
||||
bridge vlan add dev $swp2 vid 85
|
||||
bridge vlan add dev $swp1 vid 65
|
||||
|
||||
tc qdisc add dev $swp1 clsact
|
||||
tc qdisc add dev $swp2 clsact
|
||||
}
|
||||
|
||||
switch_destroy()
|
||||
{
|
||||
tc qdisc del dev $swp2 clsact
|
||||
tc qdisc del dev $swp1 clsact
|
||||
|
||||
bridge vlan del vid 65 dev $swp1
|
||||
bridge vlan del vid 85 dev $swp2
|
||||
|
||||
bridge vlan del vid 65 dev $swp2
|
||||
bridge vlan del vid 85 dev $swp1
|
||||
|
||||
ip link set dev $swp2 down
|
||||
ip link set dev $swp1 down
|
||||
|
||||
ip link del dev br0
|
||||
}
|
||||
|
||||
setup_prepare()
|
||||
{
|
||||
h1=${NETIFS[p1]}
|
||||
swp1=${NETIFS[p2]}
|
||||
|
||||
swp2=${NETIFS[p3]}
|
||||
h2=${NETIFS[p4]}
|
||||
|
||||
vrf_prepare
|
||||
|
||||
h1_create
|
||||
h2_create
|
||||
|
||||
switch_create
|
||||
}
|
||||
|
||||
cleanup()
|
||||
{
|
||||
pre_cleanup
|
||||
|
||||
switch_destroy
|
||||
|
||||
h2_destroy
|
||||
h1_destroy
|
||||
|
||||
vrf_cleanup
|
||||
}
|
||||
|
||||
vlan_modify_ingress()
|
||||
{
|
||||
RET=0
|
||||
|
||||
ping_do $h1.85 192.0.2.18
|
||||
check_fail $? "ping between two different vlans passed when should not"
|
||||
|
||||
ping6_do $h1.85 2001:db8:2::2
|
||||
check_fail $? "ping6 between two different vlans passed when should not"
|
||||
|
||||
tc filter add dev $swp1 ingress protocol all pref 1 handle 1 \
|
||||
flower action vlan modify id 65
|
||||
tc filter add dev $swp2 ingress protocol all pref 1 handle 1 \
|
||||
flower action vlan modify id 85
|
||||
|
||||
ping_do $h1.85 192.0.2.18
|
||||
check_err $? "ping between two different vlans failed when should not"
|
||||
|
||||
ping6_do $h1.85 2001:db8:2::2
|
||||
check_err $? "ping6 between two different vlans failed when should not"
|
||||
|
||||
log_test "VLAN modify at ingress"
|
||||
|
||||
tc filter del dev $swp2 ingress protocol all pref 1 handle 1 flower
|
||||
tc filter del dev $swp1 ingress protocol all pref 1 handle 1 flower
|
||||
}
|
||||
|
||||
vlan_modify_egress()
|
||||
{
|
||||
RET=0
|
||||
|
||||
ping_do $h1.85 192.0.2.18
|
||||
check_fail $? "ping between two different vlans passed when should not"
|
||||
|
||||
ping6_do $h1.85 2001:db8:2::2
|
||||
check_fail $? "ping6 between two different vlans passed when should not"
|
||||
|
||||
tc filter add dev $swp1 egress protocol all pref 1 handle 1 \
|
||||
flower action vlan modify id 85
|
||||
tc filter add dev $swp2 egress protocol all pref 1 handle 1 \
|
||||
flower action vlan modify id 65
|
||||
|
||||
ping_do $h1.85 192.0.2.18
|
||||
check_err $? "ping between two different vlans failed when should not"
|
||||
|
||||
ping6_do $h1.85 2001:db8:2::2
|
||||
check_err $? "ping6 between two different vlans failed when should not"
|
||||
|
||||
log_test "VLAN modify at egress"
|
||||
|
||||
tc filter del dev $swp2 egress protocol all pref 1 handle 1 flower
|
||||
tc filter del dev $swp1 egress protocol all pref 1 handle 1 flower
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
setup_prepare
|
||||
setup_wait
|
||||
|
||||
tests_run
|
||||
|
||||
exit $EXIT_STATUS
|
||||
Loading…
x
Reference in New Issue
Block a user