From eba852cc98714fa383a058c4da73a9d8854e8d9b Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Thu, 30 Apr 2015 10:39:54 +1200 Subject: [PATCH] KCC: write dot files in a deterministic, user specified place We were using randomised tempfile names in /tmp, initially to avoid overwriting previous runs so as to track progress. Now we hardly ever care about the old versions, and a user-specified name will be handy for testing. Signed-off-by: Douglas Bagnall Reviewed-by: Garming Sam Reviewed-by: Andrew Bartlett --- python/samba/kcc/__init__.py | 39 +++++++++++++++++---------------- python/samba/kcc/graph.py | 19 ++++++++-------- python/samba/kcc/graph_utils.py | 29 ++++++++++++++---------- source4/scripting/bin/samba_kcc | 17 +++++++------- 4 files changed, 55 insertions(+), 49 deletions(-) diff --git a/python/samba/kcc/__init__.py b/python/samba/kcc/__init__.py index f6a54979577..22804c1798a 100644 --- a/python/samba/kcc/__init__.py +++ b/python/samba/kcc/__init__.py @@ -97,10 +97,10 @@ class KCC(object): :param read_only: Don't write to the database. :param verify: Check topological invariants for the generated graphs :param debug: Write verbosely to stderr. - "param dot_files: write Graphviz files in /tmp showing topology + "param dot_file_dir: write diagnostic Graphviz files in this directory """ - def __init__(self, unix_now, readonly=False,verify=False, debug=False, - dot_files=False): + def __init__(self, unix_now, readonly=False, verify=False, debug=False, + dot_file_dir=None): """Initializes the partitions class which can hold our local DCs partitions or all the partitions in the forest @@ -140,7 +140,7 @@ class KCC(object): self.readonly = readonly self.verify = verify self.debug = debug - self.dot_files = dot_files + self.dot_file_dir = dot_file_dir def load_all_transports(self): """Loads the inter-site transport objects for Sites @@ -1158,7 +1158,7 @@ class KCC(object): g = setup_graph(part, self.site_table, self.transport_table, self.sitelink_table, bridges_required) - if self.verify or self.dot_files: + if self.verify or self.dot_file_dir is not None: dot_edges = [] for edge in g.edges: for a, b in itertools.combinations(edge.vertices, 2): @@ -1168,7 +1168,7 @@ class KCC(object): label=self.my_dsa_dnstr, properties=verify_properties, debug=DEBUG, verify=self.verify, - dot_files=self.dot_files) + dot_file_dir=self.dot_file_dir) return g @@ -2238,7 +2238,7 @@ class KCC(object): DEBUG('\n'.join(str((x.rep_dsa_guid, x.rep_dsa_dnstr)) for x in r_list)) - do_dot_files = self.dot_files and self.debug + do_dot_files = self.dot_file_dir is not None and self.debug if self.verify or do_dot_files: dot_edges = [] dot_vertices = set() @@ -2255,7 +2255,8 @@ class KCC(object): nc_x.nc_dnstr), properties=verify_properties, debug=DEBUG, verify=self.verify, - dot_files=do_dot_files, directed=True) + dot_file_dir=self.dot_file_dir, + directed=True) # For each existing nTDSConnection object implying an edge # from rj of R to ri such that j != i, an edge from rj to ri @@ -2333,7 +2334,8 @@ class KCC(object): nc_x.nc_dnstr), properties=verify_properties, debug=DEBUG, verify=self.verify, - dot_files=do_dot_files, directed=True) + dot_file_dir=self.dot_file_dir, + directed=True) def intrasite(self): """The head method for generating the intra-site KCC replica @@ -2457,8 +2459,7 @@ class KCC(object): def plot_all_connections(self, basename, verify_properties=()): verify = verify_properties and self.verify - plot = self.dot_files - if not (verify or plot): + if not verify and self.dot_file_dir is None: return dot_edges = [] @@ -2481,7 +2482,7 @@ class KCC(object): verify_and_dot(basename, dot_edges, vertices=dot_vertices, label=self.my_dsa_dnstr, properties=verify_properties, - debug=DEBUG, verify=verify, dot_files=plot, + debug=DEBUG, verify=verify, dot_file_dir=self.dot_file_dir, directed=True, edge_colors=edge_colours, vertex_colors=vertex_colours) @@ -2516,7 +2517,7 @@ class KCC(object): self.load_all_transports() self.load_all_sitelinks() - if self.verify or self.dot_files: + if self.verify or self.dot_file_dir is not None: guid_to_dnstr = {} for site in self.site_table.values(): guid_to_dnstr.update((str(dsa.dsa_guid), dnstr) @@ -2534,7 +2535,7 @@ class KCC(object): verify_and_dot('dsa_repsFrom_initial', dot_edges, directed=True, label=self.my_dsa_dnstr, properties=(), debug=DEBUG, verify=self.verify, - dot_files=self.dot_files) + dot_file_dir=self.dot_file_dir) dot_edges = [] for site in self.site_table.values(): @@ -2550,7 +2551,7 @@ class KCC(object): verify_and_dot('dsa_repsFrom_initial_all', dot_edges, directed=True, label=self.my_dsa_dnstr, properties=(), debug=DEBUG, verify=self.verify, - dot_files=self.dot_files) + dot_file_dir=self.dot_file_dir) dot_edges = [] for link in self.sitelink_table.values(): @@ -2561,7 +2562,7 @@ class KCC(object): directed=False, label=self.my_dsa_dnstr, properties=properties, debug=DEBUG, verify=self.verify, - dot_files=self.dot_files) + dot_file_dir=self.dot_file_dir) if forget_local_links: for dsa in self.my_site.dsa_table.values(): @@ -2615,7 +2616,7 @@ class KCC(object): # Step 7 self.update_rodc_connection() - if self.verify or self.dot_files: + if self.verify or self.dot_file_dir is not None: self.plot_all_connections('dsa_final', ('connected', 'forest_of_rings')) @@ -2635,7 +2636,7 @@ class KCC(object): verify_and_dot('dsa_repsFrom_final', dot_edges, directed=True, label=self.my_dsa_dnstr, properties=(), debug=DEBUG, verify=self.verify, - dot_files=self.dot_files, + dot_file_dir=self.dot_file_dir, edge_colors=edge_colors) dot_edges = [] @@ -2652,7 +2653,7 @@ class KCC(object): verify_and_dot('dsa_repsFrom_final_all', dot_edges, directed=True, label=self.my_dsa_dnstr, properties=(), debug=DEBUG, verify=self.verify, - dot_files=self.dot_files) + dot_file_dir=self.dot_file_dir) except: raise diff --git a/python/samba/kcc/graph.py b/python/samba/kcc/graph.py index ee4787452bd..9a2cc6e18e7 100644 --- a/python/samba/kcc/graph.py +++ b/python/samba/kcc/graph.py @@ -95,7 +95,7 @@ def combine_repl_info(info_a, info_b, info_c): def get_spanning_tree_edges(graph, my_site, label=None, verify=False, - dot_files=False): + dot_file_dir=None): # Phase 1: Run Dijkstra's to get a list of internal edges, which are # just the shortest-paths connecting colored vertices @@ -112,7 +112,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, for v in e.vertices: v.edges.append(e) - if verify or dot_files: + if verify or dot_file_dir is not None: graph_edges = [(a.site.site_dnstr, b.site.site_dnstr) for a, b in itertools.chain( @@ -120,7 +120,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, for edge in e_set.edges))] graph_nodes = [v.site.site_dnstr for v in graph.vertices] - if dot_files: + if dot_file_dir is not None: write_dot_file('edgeset_%s' % (edgeType,), graph_edges, vertices=graph_nodes, label=label) @@ -148,15 +148,14 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, setup_vertices(graph) process_edge_set(graph, None, internal_edges) - if verify or dot_files: + if verify or dot_file_dir is not None: graph_edges = [(e.v1.site.site_dnstr, e.v2.site.site_dnstr) for e in internal_edges] graph_nodes = [v.site.site_dnstr for v in graph.vertices] verify_properties = ('multi_edge_forest',) verify_and_dot('prekruskal', graph_edges, graph_nodes, label=label, properties=verify_properties, debug=DEBUG, - verify=verify, - dot_files=dot_files) + verify=verify, dot_file_dir=dot_file_dir) # Phase 2: Run Kruskal's on the internal edges output_edges, components = kruskal(graph, internal_edges) @@ -172,7 +171,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, else: v.dist_to_red = v.repl_info.cost - if verify or dot_files: + if verify or dot_file_dir is not None: graph_edges = [(e.v1.site.site_dnstr, e.v2.site.site_dnstr) for e in internal_edges] graph_nodes = [v.site.site_dnstr for v in graph.vertices] @@ -180,7 +179,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, verify_and_dot('postkruskal', graph_edges, graph_nodes, label=label, properties=verify_properties, debug=DEBUG, verify=verify, - dot_files=dot_files) + dot_file_dir=dot_file_dir) # Ensure only one-way connections for partial-replicas, # and make sure they point the right way. @@ -198,7 +197,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, edge.vertices[:] = w, v edge_list.append(edge) - if verify or dot_files: + if verify or dot_file_dir is not None: graph_edges = [[x.site.site_dnstr for x in e.vertices] for e in edge_list] #add the reverse edge if not directed. @@ -211,7 +210,7 @@ def get_spanning_tree_edges(graph, my_site, label=None, verify=False, label=label, properties=verify_properties, debug=DEBUG, verify=verify, directed=True, - dot_files=dot_files) + dot_file_dir=dot_file_dir) # count the components return edge_list, components diff --git a/python/samba/kcc/graph_utils.py b/python/samba/kcc/graph_utils.py index 5fea5f0152e..69ba6d3d5fc 100644 --- a/python/samba/kcc/graph_utils.py +++ b/python/samba/kcc/graph_utils.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os import itertools from samba.kcc.debug import null_debug, PURPLE, MAGENTA, DARK_YELLOW, RED @@ -26,14 +27,15 @@ from samba.kcc.debug import DARK_GREEN, C_NORMAL, GREY def write_dot_file(basename, edge_list, vertices=None, label=None, - destdir=None, reformat_labels=True, directed=False, + dot_file_dir=None, reformat_labels=True, directed=False, debug=None, edge_colors=None, edge_labels=None, vertex_colors=None): - from tempfile import NamedTemporaryFile if label: - basename += '_' + label.translate(None, ', ') # fix DN, guid labels - f = NamedTemporaryFile(suffix='.dot', prefix=basename + '_', delete=False, - dir=destdir) + # sanitise DN and guid labels + basename += '_' + label.translate(None, ', ') + + f = open(os.path.join(dot_file_dir, "%s.dot" % basename), 'w') + if debug is not None: debug(f.name) graphname = ''.join(x for x in basename if x.isalnum()) @@ -341,19 +343,22 @@ def verify_graph(title, edges, vertices=None, directed=False, properties=(), debug(C_NORMAL) -def verify_and_dot(basename, edges, vertices=None, label=None, destdir=None, - reformat_labels=True, directed=False, properties=(), - fatal=True, debug=None, verify=True, dot_files=False, - edge_colors=None, edge_labels=None, vertex_colors=None): +def verify_and_dot(basename, edges, vertices=None, label=None, + reformat_labels=True, directed=False, + properties=(), fatal=True, debug=None, + verify=True, dot_file_dir=None, + edge_colors=None, edge_labels=None, + vertex_colors=None): title = '%s %s' % (basename, label or '') if verify: verify_graph(title, edges, vertices, properties=properties, fatal=fatal, debug=debug) - if dot_files: + if dot_file_dir is not None: write_dot_file(basename, edges, vertices=vertices, label=label, - destdir=destdir, reformat_labels=reformat_labels, - directed=directed, debug=debug, edge_colors=edge_colors, + dot_file_dir=dot_file_dir, + reformat_labels=reformat_labels, directed=directed, + debug=debug, edge_colors=edge_colors, edge_labels=edge_labels, vertex_colors=vertex_colors) diff --git a/source4/scripting/bin/samba_kcc b/source4/scripting/bin/samba_kcc index b85e4679450..5e3d363b7b5 100755 --- a/source4/scripting/bin/samba_kcc +++ b/source4/scripting/bin/samba_kcc @@ -55,7 +55,8 @@ from samba.kcc import KCC def test_all_reps_from(lp, creds, unix_now, rng_seed=None): # This implies readonly and attempt_live_connections kcc = KCC(unix_now, readonly=True, - verify=opts.verify, debug=opts.debug, dot_files=opts.dot_files) + verify=opts.verify, debug=opts.debug, + dot_file_dir=opts.dot_file_dir) kcc.load_samdb(opts.dburl, lp, creds) dsas = kcc.list_dsas() needed_parts = {} @@ -76,7 +77,7 @@ def test_all_reps_from(lp, creds, unix_now, rng_seed=None): random.seed(rng_seed) kcc = KCC(unix_now, readonly=True, verify=opts.verify, debug=opts.debug, - dot_files=opts.dot_files) + dot_file_dir=opts.dot_file_dir) kcc.run(opts.dburl, lp, creds, forced_local_dsa=dsa_dn, forget_local_links=opts.forget_local_links, forget_intersite_links=opts.forget_intersite_links, @@ -123,7 +124,8 @@ def test_all_reps_from(lp, creds, unix_now, rng_seed=None): verify_and_dot('all-dsa-connections', dot_edges, vertices=dot_vertices, label="all dsa NTDSConnections", properties=(), - debug=DEBUG, verify=opts.verify, dot_files=opts.dot_files, + debug=DEBUG, verify=opts.verify, + dot_file_dir=opts.dot_file_dir, directed=True, edge_colors=colours, vertex_colors=vertex_colours) @@ -133,7 +135,7 @@ def test_all_reps_from(lp, creds, unix_now, rng_seed=None): verify_and_dot('all-repsFrom_%s__%s' % (name, part), edges, directed=True, label=part, properties=(), debug=DEBUG, verify=opts.verify, - dot_files=opts.dot_files) + dot_file_dir=opts.dot_file_dir) ################################################## # samba_kcc entry point @@ -165,9 +167,8 @@ parser.add_option("--list-verify-tests", "and do nothing else"), action="store_true") -parser.add_option("--no-dot-files", dest='dot_files', - help="Don't write dot graph files in /tmp", - default=True, action="store_false") +parser.add_option("--dot-file-dir", default=None, + help="Write Graphviz .dot files to this directory") parser.add_option("--seed", help="random number seed", @@ -269,7 +270,7 @@ if opts.test_all_reps_from: # Instantiate Knowledge Consistency Checker and perform run kcc = KCC(unix_now, readonly=opts.readonly, verify=opts.verify, - debug=opts.debug, dot_files=opts.dot_files) + debug=opts.debug, dot_file_dir=opts.dot_file_dir) if opts.exportldif: