mirror of
https://github.com/samba-team/samba.git
synced 2025-01-03 01:18:10 +03:00
traffic: new version of model with packet_rate, version number
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz> Reviewed-by: Andrew Bartlett <abartlet@samba.org>
This commit is contained in:
Douglas Bagnall
Douglas Bagnall
parent
273eb3dffb
commit
7edf58dc58
@ -54,6 +54,8 @@ from samba.compat import get_string
|
||||
from samba.logger import get_samba_logger
|
||||
import bisect
|
||||
|
||||
CURRENT_MODEL_VERSION = 2 # save as this
|
||||
REQUIRED_MODEL_VERSION = 2 # load accepts this or greater
|
||||
SLEEP_OVERHEAD = 3e-4
|
||||
|
||||
# we don't use None, because it complicates [de]serialisation
|
||||
@ -1136,7 +1138,7 @@ class TrafficModel(object):
|
||||
self.n = n
|
||||
self.dns_opcounts = defaultdict(int)
|
||||
self.cumulative_duration = 0.0
|
||||
self.conversation_rate = [0, 1]
|
||||
self.packet_rate = [0, 1]
|
||||
|
||||
def learn(self, conversations, dns_opcounts={}):
|
||||
prev = 0.0
|
||||
@ -1149,10 +1151,15 @@ class TrafficModel(object):
|
||||
self.dns_opcounts[k] += v
|
||||
|
||||
if len(conversations) > 1:
|
||||
elapsed =\
|
||||
conversations[-1].start_time - conversations[0].start_time
|
||||
self.conversation_rate[0] = len(conversations)
|
||||
self.conversation_rate[1] = elapsed
|
||||
first = conversations[0].start_time
|
||||
total = 0
|
||||
last = first + 0.1
|
||||
for c in conversations:
|
||||
total += len(c)
|
||||
last = max(last, c.packets[-1].timestamp)
|
||||
|
||||
self.packet_rate[0] = total
|
||||
self.packet_rate[1] = last - first
|
||||
|
||||
for c in conversations:
|
||||
client, server = c.guess_client_server(server)
|
||||
@ -1196,7 +1203,8 @@ class TrafficModel(object):
|
||||
'ngrams': ngrams,
|
||||
'query_details': query_details,
|
||||
'cumulative_duration': self.cumulative_duration,
|
||||
'conversation_rate': self.conversation_rate,
|
||||
'packet_rate': self.packet_rate,
|
||||
'version': CURRENT_MODEL_VERSION
|
||||
}
|
||||
d['dns'] = self.dns_opcounts
|
||||
|
||||
@ -1211,6 +1219,17 @@ class TrafficModel(object):
|
||||
|
||||
d = json.load(f)
|
||||
|
||||
try:
|
||||
version = d["version"]
|
||||
if version < REQUIRED_MODEL_VERSION:
|
||||
raise ValueError("the model file is version %d; "
|
||||
"version %d is required" %
|
||||
(version, REQUIRED_MODEL_VERSION))
|
||||
except KeyError:
|
||||
raise ValueError("the model file lacks a version number; "
|
||||
"version %d is required" %
|
||||
(REQUIRED_MODEL_VERSION))
|
||||
|
||||
for k, v in d['ngrams'].items():
|
||||
k = tuple(str(k).split('\t'))
|
||||
values = self.ngrams.setdefault(k, [])
|
||||
@ -1232,18 +1251,22 @@ class TrafficModel(object):
|
||||
self.dns_opcounts[k] += v
|
||||
|
||||
self.cumulative_duration = d['cumulative_duration']
|
||||
self.conversation_rate = d['conversation_rate']
|
||||
|
||||
def construct_conversation(self, timestamp=0.0, client=2, server=1,
|
||||
hard_stop=None, replay_speed=1):
|
||||
"""Construct a individual converation from the model."""
|
||||
|
||||
c = Conversation(timestamp, (server, client), conversation_id=client)
|
||||
self.packet_rate = d['packet_rate']
|
||||
|
||||
def construct_conversation_sequence(self, timestamp=0.0,
|
||||
hard_stop=None,
|
||||
replay_speed=1,
|
||||
ignore_before=0):
|
||||
"""Construct an individual conversation packet sequence from the
|
||||
model.
|
||||
"""
|
||||
c = []
|
||||
key = (NON_PACKET,) * (self.n - 1)
|
||||
if ignore_before is None:
|
||||
ignore_before = timestamp - 1
|
||||
|
||||
while key in self.ngrams:
|
||||
p = random.choice(self.ngrams.get(key, NON_PACKET))
|
||||
while True:
|
||||
p = random.choice(self.ngrams.get(key, (NON_PACKET,)))
|
||||
if p == NON_PACKET:
|
||||
break
|
||||
|
||||
@ -1263,47 +1286,50 @@ class TrafficModel(object):
|
||||
timestamp += wait
|
||||
if hard_stop is not None and timestamp > hard_stop:
|
||||
break
|
||||
c.add_short_packet(timestamp, protocol, opcode, extra)
|
||||
if timestamp >= ignore_before:
|
||||
c.append((timestamp, protocol, opcode, extra))
|
||||
|
||||
key = key[1:] + (p,)
|
||||
|
||||
return c
|
||||
|
||||
def generate_conversations(self, rate, duration, replay_speed=1):
|
||||
def generate_conversations(self, scale, duration, replay_speed=1,
|
||||
server=1, client=2):
|
||||
"""Generate a list of conversations from the model."""
|
||||
|
||||
# We run the simulation for at least ten times as long as our
|
||||
# desired duration, and take a section near the start.
|
||||
rate_n, rate_t = self.conversation_rate
|
||||
|
||||
duration2 = max(rate_t, duration * 2)
|
||||
n = rate * duration2 * rate_n / rate_t
|
||||
|
||||
server = 1
|
||||
client = 2
|
||||
# We run the simulation for ten times as long as our desired
|
||||
# duration, and take the section at the end.
|
||||
lead_in = 9 * duration
|
||||
rate_n, rate_t = self.packet_rate
|
||||
target_packets = int(duration * scale * rate_n / rate_t)
|
||||
|
||||
conversations = []
|
||||
end = duration2
|
||||
start = end - duration
|
||||
n_packets = 0
|
||||
|
||||
while client < n + 2:
|
||||
start = random.uniform(0, duration2)
|
||||
c = self.construct_conversation(start,
|
||||
client,
|
||||
server,
|
||||
hard_stop=(duration2 * 5),
|
||||
replay_speed=replay_speed)
|
||||
while n_packets < target_packets:
|
||||
start = random.uniform(-lead_in, duration)
|
||||
c = self.construct_conversation_sequence(start,
|
||||
hard_stop=duration,
|
||||
replay_speed=replay_speed,
|
||||
ignore_before=0)
|
||||
conversations.append(c)
|
||||
n_packets += len(c)
|
||||
|
||||
c.forget_packets_outside_window(start, end)
|
||||
c.renormalise_times(start)
|
||||
if len(c) != 0:
|
||||
conversations.append(c)
|
||||
print(("we have %d packets (target %d) in %d conversations at scale %f"
|
||||
% (n_packets, target_packets, len(conversations), scale)),
|
||||
file=sys.stderr)
|
||||
conversations.sort() # sorts by first element == start time
|
||||
return seq_to_conversations(conversations)
|
||||
|
||||
|
||||
def seq_to_conversations(seq, server=1, client=2):
|
||||
conversations = []
|
||||
for s in seq:
|
||||
if s:
|
||||
c = Conversation(s[0][0], (server, client), s)
|
||||
client += 1
|
||||
|
||||
print(("we have %d conversations at rate %f" %
|
||||
(len(conversations), rate)), file=sys.stderr)
|
||||
conversations.sort()
|
||||
return conversations
|
||||
conversations.append(c)
|
||||
return conversations
|
||||
|
||||
|
||||
IP_PROTOCOLS = {
|
||||
|
||||
@ -30,14 +30,14 @@
|
||||
"-": 1
|
||||
}
|
||||
},
|
||||
"conversation_rate": [
|
||||
2,
|
||||
0.12712717056274414
|
||||
],
|
||||
"dns": {
|
||||
"1": 9,
|
||||
"0": 9
|
||||
},
|
||||
"packet_rate": [
|
||||
50,
|
||||
0.32482
|
||||
],
|
||||
"query_details": {
|
||||
"rpc_netlogon:29": {
|
||||
"-": 1
|
||||
@ -56,5 +56,6 @@
|
||||
"": 1
|
||||
}
|
||||
},
|
||||
"cumulative_duration": 0.39243292808532715
|
||||
"cumulative_duration": 0.39243292808532715,
|
||||
"version": 2
|
||||
}
|
||||
@ -1,4 +1,26 @@
|
||||
{
|
||||
"packet_rate": [
|
||||
10,
|
||||
0.22707200050354004
|
||||
],
|
||||
"query_details": {
|
||||
"rpc_netlogon:29": {
|
||||
"-": 1
|
||||
},
|
||||
"cldap:3": {
|
||||
"\t\t\tNetlogon\t\t\t": 3
|
||||
},
|
||||
"ldap:3": {
|
||||
"\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1,
|
||||
"2\tDC,DC\t\tcn\t\t\t": 1
|
||||
},
|
||||
"ldap:2": {
|
||||
"\t\t\t\t\t\t": 1
|
||||
},
|
||||
"kerberos:": {
|
||||
"": 1
|
||||
}
|
||||
},
|
||||
"ngrams": {
|
||||
"-\t-": {
|
||||
"cldap:3": 1,
|
||||
@ -30,31 +52,10 @@
|
||||
"-": 1
|
||||
}
|
||||
},
|
||||
"conversation_rate": [
|
||||
2,
|
||||
0.12712717056274414
|
||||
],
|
||||
"version": 2,
|
||||
"dns": {
|
||||
"1": 9,
|
||||
"0": 9
|
||||
},
|
||||
"query_details": {
|
||||
"rpc_netlogon:29": {
|
||||
"-": 1
|
||||
},
|
||||
"cldap:3": {
|
||||
"\t\t\tNetlogon\t\t\t": 3
|
||||
},
|
||||
"ldap:3": {
|
||||
"\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1,
|
||||
"2\tDC,DC\t\tcn\t\t\t": 1
|
||||
},
|
||||
"ldap:2": {
|
||||
"\t\t\t\t\t\t": 1
|
||||
},
|
||||
"kerberos:": {
|
||||
"": 1
|
||||
}
|
||||
},
|
||||
"cumulative_duration": 0.39243292808532715
|
||||
}
|
||||
18
python/samba/tests/blackbox/testdata/traffic_replay-0.expected
vendored
Normal file
18
python/samba/tests/blackbox/testdata/traffic_replay-0.expected
vendored
Normal file
@ -0,0 +1,18 @@
|
||||
0.011388 06 2 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.221447 06 2 1 ldap 2 unbindRequest
|
||||
0.460878 06 3 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.581933 11 4 1 cldap 3 searchRequest Netlogon
|
||||
0.596977 11 4 1 cldap 3 searchRequest Netlogon
|
||||
0.611184 11 4 1 cldap 3 searchRequest Netlogon
|
||||
0.666808 06 3 1 ldap 2 unbindRequest
|
||||
0.744297 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.768994 06 4 1 kerberos
|
||||
0.772476 06 4 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.805442 11 5 1 cldap 3 searchRequest Netlogon
|
||||
0.805536 11 5 1 cldap 3 searchRequest Netlogon
|
||||
0.807659 11 5 1 cldap 3 searchRequest Netlogon
|
||||
0.808614 11 5 1 cldap 3 searchRequest Netlogon
|
||||
0.808819 11 5 1 cldap 3 searchRequest Netlogon
|
||||
0.865384 06 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
|
||||
0.973595 06 5 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.974012 06 5 1 kerberos
|
||||
19
python/samba/tests/blackbox/testdata/traffic_replay-1.expected
vendored
Normal file
19
python/samba/tests/blackbox/testdata/traffic_replay-1.expected
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
0.011519 11 2 1 cldap 3 searchRequest Netlogon
|
||||
0.012916 11 2 1 cldap 3 searchRequest Netlogon
|
||||
0.158388 06 3 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.164506 06 2 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.166151 06 2 1 kerberos
|
||||
0.166301 06 2 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
|
||||
0.258932 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.259908 06 4 1 kerberos
|
||||
0.260073 06 4 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.286044 06 5 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.295757 06 3 1 ldap 2 unbindRequest
|
||||
0.459791 06 5 1 ldap 2 unbindRequest
|
||||
0.553887 06 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
|
||||
0.641127 11 7 1 cldap 3 searchRequest Netlogon
|
||||
0.641297 11 7 1 cldap 3 searchRequest Netlogon
|
||||
0.783989 06 6 1 ldap 2 unbindRequest
|
||||
0.901096 06 7 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.915260 06 7 1 kerberos
|
||||
0.915711 06 7 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
@ -1,10 +1,18 @@
|
||||
0.000000 06 3 5 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.127127 11 6 1 cldap 3 searchRequest Netlogon
|
||||
0.127391 11 6 1 cldap 3 searchRequest Netlogon
|
||||
0.127689 11 6 1 cldap 3 searchRequest Netlogon
|
||||
0.227072 06 3 5 1 ldap 2 unbindRequest
|
||||
0.282995 06 16 6 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.287553 06 18 6 1 kerberos
|
||||
0.287759 06 16 1 6 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.291161 06 19 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
|
||||
0.292488 06 18 1 6 kerberos
|
||||
0.040433 06 2 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.059203 06 2 1 kerberos
|
||||
0.061641 06 2 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.535074 11 3 1 cldap 3 searchRequest Netlogon
|
||||
0.535369 11 3 1 cldap 3 searchRequest Netlogon
|
||||
0.536671 11 3 1 cldap 3 searchRequest Netlogon
|
||||
0.537238 11 3 1 cldap 3 searchRequest Netlogon
|
||||
0.537362 11 3 1 cldap 3 searchRequest Netlogon
|
||||
0.602824 11 4 1 cldap 3 searchRequest Netlogon
|
||||
0.640115 11 4 1 cldap 3 searchRequest Netlogon
|
||||
0.714546 06 3 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.715865 06 3 1 kerberos
|
||||
0.716613 06 3 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.767674 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
|
||||
0.778022 06 5 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.792356 06 4 1 kerberos
|
||||
0.792763 06 4 1 ldap 3 searchRequest 2 DC,DC cn
|
||||
0.960412 06 5 1 ldap 2 unbindRequest
|
||||
|
||||
@ -75,4 +75,4 @@ class TrafficLearnerTests(BlackboxTestCase):
|
||||
expected_details = {k: sorted(v) for k, v in expected.query_details.items()}
|
||||
self.assertEquals(expected_details, actual_details)
|
||||
self.assertEquals(expected.cumulative_duration, actual.cumulative_duration)
|
||||
self.assertEquals(expected.conversation_rate, actual.conversation_rate)
|
||||
self.assertEquals(expected.packet_rate, actual.packet_rate)
|
||||
|
||||
@ -25,12 +25,13 @@ from samba.tests import BlackboxTestCase
|
||||
|
||||
DATA_DIR = "python/samba/tests/blackbox/testdata"
|
||||
SCRIPT = "script/traffic_replay"
|
||||
FIXED = "--fixed-password trafficreplay01%"
|
||||
FIXED = "--fixed-password=trafficreplay01%"
|
||||
SERVER = os.environ["SERVER"]
|
||||
PASSWORD = os.environ["PASSWORD"]
|
||||
USER = os.environ["USERNAME"]
|
||||
STD_OPTIONS = "-U%s%%%s %s" % (USER, PASSWORD, SERVER)
|
||||
EXPECTED_NAME = os.path.join(DATA_DIR, "traffic_replay.expected")
|
||||
CREDS = "-U%s%%%s" % (USER, PASSWORD)
|
||||
MODEL = os.path.join(DATA_DIR, "traffic-sample-very-short.model")
|
||||
EXPECTED_OUTPUT = os.path.join(DATA_DIR, "traffic_replay-%s.expected")
|
||||
|
||||
|
||||
@contextmanager
|
||||
@ -49,7 +50,7 @@ class TrafficLearnerTests(BlackboxTestCase):
|
||||
|
||||
def tearDown(self):
|
||||
options = "--clean-up"
|
||||
command = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
|
||||
command = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
|
||||
self.check_run(command)
|
||||
|
||||
def test_generate_users_only(self):
|
||||
@ -57,16 +58,48 @@ class TrafficLearnerTests(BlackboxTestCase):
|
||||
"""
|
||||
options = ("--generate-users-only --number-of-users 20 "
|
||||
"--number-of-groups 5 --average-groups-per-user 2")
|
||||
command = "%s %s %s %s" % (
|
||||
SCRIPT, options, FIXED, STD_OPTIONS)
|
||||
command = "%s %s %s %s %s" % (
|
||||
SCRIPT, options, FIXED, CREDS, SERVER)
|
||||
self.check_run(command)
|
||||
command = "%s %s %s %s %s %s" % (
|
||||
SCRIPT, MODEL, options, FIXED, CREDS, SERVER)
|
||||
self.check_run(command)
|
||||
|
||||
def test_summary_generation(self):
|
||||
"""Ensure a summary file is generated and the contents are correct"""
|
||||
|
||||
for i, opts in enumerate((["--random-seed=3"],
|
||||
["--random-seed=4"],
|
||||
["--random-seed=3",
|
||||
"--conversation-persistence=0.5"],
|
||||
["--random-seed=3",
|
||||
"--old-scale",
|
||||
"--conversation-persistence=0.95"],
|
||||
)):
|
||||
with temp_file(self.tempdir) as output:
|
||||
command = ([SCRIPT, MODEL,
|
||||
"--traffic-summary", output,
|
||||
"-D1", "-S0.1"] +
|
||||
opts +
|
||||
[FIXED, CREDS, SERVER])
|
||||
self.check_run(command)
|
||||
expected = open(EXPECTED_OUTPUT % i).read()
|
||||
actual = open(output).read()
|
||||
self.assertStringsEqual(expected, actual)
|
||||
|
||||
def test_summary_replay_no_fixed(self):
|
||||
"""Ensure a summary file with no fixed password fails
|
||||
"""
|
||||
command = [SCRIPT, MODEL, CREDS, SERVER]
|
||||
self.check_exit_code(command, 1)
|
||||
|
||||
def test_model_replay(self):
|
||||
"""Ensure a model can be replayed against a DC
|
||||
"""
|
||||
|
||||
model = "testdata/traffic-sample-very-short.model"
|
||||
command = "%s %s-D 5 %s %s" % (SCRIPT, model, FIXED, STD_OPTIONS)
|
||||
command = [SCRIPT, MODEL,
|
||||
FIXED,
|
||||
'-D2', '-S0.1',
|
||||
CREDS, SERVER]
|
||||
self.check_run(command)
|
||||
|
||||
def test_generate_users_only_no_password(self):
|
||||
@ -74,5 +107,7 @@ class TrafficLearnerTests(BlackboxTestCase):
|
||||
"""
|
||||
options = ("--generate-users-only --number-of-users 20 "
|
||||
"--number-of-groups 5 --average-groups-per-user 2")
|
||||
command = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
|
||||
command = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
|
||||
self.check_exit_code(command, 1)
|
||||
command = "%s %s %s %s %s" % (SCRIPT, MODEL, options, CREDS, SERVER)
|
||||
self.check_exit_code(command, 1)
|
||||
|
||||
@ -235,7 +235,7 @@ def main():
|
||||
sys.exit()
|
||||
|
||||
# ingest the model
|
||||
if model_file:
|
||||
if model_file and not opts.generate_users_only:
|
||||
try:
|
||||
model = traffic.TrafficModel()
|
||||
|
||||
|
||||
2
selftest/knownfail.d/traffic
Normal file
2
selftest/knownfail.d/traffic
Normal file
@ -0,0 +1,2 @@
|
||||
# This is in flux, so lets fail it for a while
|
||||
samba.tests.blackbox.traffic_replay.samba.tests.blackbox.traffic_replay.TrafficLearnerTests.test_summary_generation
|
||||
Loading…
Reference in New Issue
Block a user