diff --git a/Makefile.in b/Makefile.in index 37eaec513..785cb0f53 100644 --- a/Makefile.in +++ b/Makefile.in @@ -127,3 +127,23 @@ lcov: $(LCOV_TRACES) endif endif + +ifeq ("$(TESTING)", "yes") +# testing and report generation +RUBY=ruby1.9 -Ireport-generators/lib -Ireport-generators/test + +.PHONEY: unit-test ruby-test test-programs + +test-programs: + +unit-test: test-programs + $(RUBY) report-generators/unit_test.rb $(shell find . -name TESTS) + $(RUBY) report-generators/title_page.rb + +memcheck: test-programs + $(RUBY) report-generators/memcheck.rb $(shell find . -name TESTS) + $(RUBY) report-generators/title_page.rb + +ruby-test: + $(RUBY) report-generators/test/ts.rb +endif \ No newline at end of file diff --git a/configure.in b/configure.in index 6b5d65a3f..db1376f99 100644 --- a/configure.in +++ b/configure.in @@ -700,6 +700,22 @@ if test "x$PROFILING" = xyes; then fi fi +################################################################################ +dnl -- Enable testing +AC_MSG_CHECKING(whether to enable unit testing) +AC_ARG_ENABLE(testing, + AC_HELP_STRING(--enable-testing, [enable testing targets in the makefile]), + TESTING=$enableval, TESTING=no) +AC_MSG_RESULT($TESTING) + +if test "$TESTING" = yes; then + AC_PATH_PROG(RUBY19, ruby1.9) + AC_PATH_PROG(VALGRIND, valgrind) + if test -z "$RUBY19" -o -z "$VALGRIND"; then + AC_MSG_ERROR([ruby1.9 and valgrind are required for testing]) + fi +fi + ################################################################################ dnl -- Disable devmapper AC_MSG_CHECKING(whether to use device-mapper) @@ -1260,6 +1276,7 @@ AC_SUBST(SELINUX_PC) AC_SUBST(SNAPSHOTS) AC_SUBST(STATICDIR) AC_SUBST(STATIC_LINK) +AC_SUBST(TESTING) AC_SUBST(UDEV_LIBS) AC_SUBST(UDEV_PC) AC_SUBST(UDEV_RULES) diff --git a/make.tmpl.in b/make.tmpl.in index 75678d547..2eb6e9b21 100644 --- a/make.tmpl.in +++ b/make.tmpl.in @@ -45,6 +45,7 @@ PTHREAD_LIBS = @PTHREAD_LIBS@ READLINE_LIBS = @READLINE_LIBS@ SELINUX_LIBS = @SELINUX_LIBS@ UDEV_LIBS = @UDEV_LIBS@ +TESTING = @TESTING@ # Setup directory variables prefix = @prefix@ diff --git a/report-generators/lib/log.rb b/report-generators/lib/log.rb new file mode 100644 index 000000000..c98b18395 --- /dev/null +++ b/report-generators/lib/log.rb @@ -0,0 +1,30 @@ +# Merely wraps the logger library with a bit of standard policy. +require 'logger' + +module Log + $log = Logger.new(STDERR) + + def init(io_) + $log = Logger.new(io_) + end +end + +def fatal(*args) + $log.fatal(*args) +end + +def error(*args) + $log.error(*args) +end + +def info(*args) + $log.info(*args) +end + +def warning(*args) + $log.warn(*args) +end + +def debug(*args) + $log.debug(*args) +end diff --git a/report-generators/lib/report_templates.rb b/report-generators/lib/report_templates.rb new file mode 100644 index 000000000..83e3acbf0 --- /dev/null +++ b/report-generators/lib/report_templates.rb @@ -0,0 +1,28 @@ +# Policy for the location of report templates +require 'string-store' + +class TemplateStringStore < StringStore + def initialize() + super(['report-generators/templates']) + end +end + +module ReportTemplates + def generate_report(report, bs, dest_path = nil) + include Reports + reports = ReportRegister.new + template_store = TemplateStringStore.new + report = reports.get_report(report) + erb = ERB.new(template_store.lookup(report.template)) + body = erb.result(bs) + title = report.short_desc + + erb = ERB.new(template_store.lookup("boiler_plate.rhtml")) + txt = erb.result(binding) + + dest_path = dest_path.nil? ? report.path : dest_path + dest_path.open("w") do |out| + out.puts txt + end + end +end diff --git a/report-generators/lib/reports.rb b/report-generators/lib/reports.rb new file mode 100644 index 000000000..573333243 --- /dev/null +++ b/report-generators/lib/reports.rb @@ -0,0 +1,48 @@ +# Data about the various reports we support +require 'log' +require 'pathname' + +module Reports + Report = Struct.new(:short_desc, :desc, :path, :template) + + class ReportRegister + attr_reader :reports + + private + def add_report(sym, *args) + @reports[sym] = Report.new(*args) + end + + public + def initialize() + @reports = Hash.new + + add_report(:unit_test, + "Unit Tests", + "unit tests", + Pathname.new("reports/unit.html"), + Pathname.new("unit_test.rhtml")) + + add_report(:memcheck, + "Memory Tests", + "unit tests with valgrind memory checking", + Pathname.new("reports/memcheck.html"), + Pathname.new("memcheck.rhtml")) + + add_report(:unit_detail, + "Unit Test Detail", + "unit test detail", + Pathname.new("reports/unit_detail.html"), # FIXME replace this with a lambda + Pathname.new("unit_detail.rhtml")) + end + + def get_report(sym) + raise RuntimeError, "unknown report '#{sym}'" unless @reports.member?(sym) + @reports[sym] + end + + def each(&block) + @reports.each(&block) + end + end +end diff --git a/report-generators/lib/schedule_file.rb b/report-generators/lib/schedule_file.rb new file mode 100644 index 000000000..1e164f450 --- /dev/null +++ b/report-generators/lib/schedule_file.rb @@ -0,0 +1,46 @@ +# Parses the simple colon delimited test schedule files. + +ScheduledTest = Struct.new(:desc, :command_line, :status, :output) + +class Schedule + attr_reader :dir, :schedules + + def initialize(dir, ss) + @dir = dir + @schedules = ss + end + + def run + Dir::chdir(@dir.to_s) do + @schedules.each do |s| + reader, writer = IO.pipe + print "#{s.desc} ... " + pid = spawn(s.command_line, [ STDERR, STDOUT ] => writer) + writer.close + _, s.status = Process::waitpid2(pid) + puts (s.status.success? ? "pass" : "fail") + s.output = reader.read + end + end + end + + def self.read(dir, io) + ss = Array.new + + io.readlines.each do |line| + case line.strip + when /^\#.*/ + next + + when /([^:]+):(.*)/ + ss << ScheduledTest.new($1.strip, $2.strip) + + else + raise RuntimeError, "badly formatted schedule line" + end + end + + Schedule.new(dir, ss) + end +end + diff --git a/report-generators/lib/string-store.rb b/report-generators/lib/string-store.rb new file mode 100644 index 000000000..36d819a95 --- /dev/null +++ b/report-generators/lib/string-store.rb @@ -0,0 +1,32 @@ +# Provides a simple way of accessing the contents of files by a symbol +# name. Useful for erb templates. + +require 'pathname' + +class StringStore + attr_accessor :path + + def initialize(p) + @paths = p.nil? ? Array.new : p # FIXME: do we need to copy p ? + end + + def lookup(sym) + files = expansions(sym) + + @paths.each do |p| + files.each do |f| + pn = Pathname.new("#{p}/#{f}") + if pn.file? + return pn.read + end + end + end + + raise RuntimeError, "unknown string entry: #{sym}" + end + + private + def expansions(sym) + ["#{sym}", "#{sym}.txt"] + end +end diff --git a/report-generators/memcheck.rb b/report-generators/memcheck.rb new file mode 100644 index 000000000..fcc8927e7 --- /dev/null +++ b/report-generators/memcheck.rb @@ -0,0 +1,76 @@ +# Reads the schedule files given on the command line. Runs them and +# generates the reports. + +# FIXME: a lot of duplication with unit_test.rb + +require 'schedule_file' +require 'pathname' +require 'reports' +require 'erb' +require 'report_templates' + +include ReportTemplates + +schedules = ARGV.map do |f| + p = Pathname.new(f) + Schedule.read(p.dirname, p) +end + +total_passed = 0 +total_failed = 0 + +# We need to make sure the lvm shared libs are in the LD_LIBRARY_PATH +ENV['LD_LIBRARY_PATH'] = `pwd`.strip + "/libdm:" + (ENV['LD_LIBRARY_PATH'] || '') + +ENV['TEST_TOOL'] = "valgrind --leak-check=full --show-reachable=yes" + +schedules.each do |s| + s.run + + s.schedules.each do |t| + if t.status.success? + total_passed += 1 + else + total_failed += 1 + end + end +end + +def mangle(txt) + txt.gsub(/\s+/, '_') +end + +MemcheckStats = Struct.new(:definitely_lost, :indirectly_lost, :possibly_lost, :reachable) + +def format(bytes, blocks) + "#{bytes} bytes, #{blocks} blocks" +end + +# Examines the output for details of leaks +def extract_stats(t) + d = i = p = r = '-' + + t.output.split("\n").each do |l| + case l + when /==\d+== definitely lost: ([0-9,]+) bytes in ([0-9,]+) blocks/ + d = format($1, $2) + when /==\d+== indirectly lost: ([0-9,]+) bytes in ([0-9,]+) blocks/ + i = format($1, $2) + when /==\d+== possibly lost: ([0-9,]+) bytes in ([0-9,]+) blocks/ + p = format($1, $2) + when /==\d+== still reachable: ([0-9,]+) bytes in ([0-9,]+) blocks/ + r = format($1, $2) + end + end + + MemcheckStats.new(d, i, p, r) +end + +generate_report(:memcheck, binding) + +# now we generate a detail report for each schedule +schedules.each do |s| + s.schedules.each do |t| + generate_report(:unit_detail, binding, Pathname.new("reports/memcheck_#{mangle(t.desc)}.html")) + end +end diff --git a/report-generators/templates/boiler_plate.rhtml b/report-generators/templates/boiler_plate.rhtml new file mode 100644 index 000000000..23f01cbdf --- /dev/null +++ b/report-generators/templates/boiler_plate.rhtml @@ -0,0 +1,25 @@ + + + +<%= title %> + + + + + +
+
+ + + + +
Generation times
Unit tests
Memory tests
+
+ +
+ <%= body %> +
+
+ diff --git a/report-generators/templates/index.rhtml b/report-generators/templates/index.rhtml new file mode 100644 index 000000000..6d72081fb --- /dev/null +++ b/report-generators/templates/index.rhtml @@ -0,0 +1,17 @@ + + +<% [:unit_test, :memcheck].each do |sym| %> +<% r = reports.get_report(sym) %> + + + + +<% end %> +
ReportGeneration time
+ <% if r.path.file? %> + <%= r.short_desc %> + <% else %> + <%= r.short_desc %> + <% end %> + <%= safe_mtime(r) %>
+ diff --git a/report-generators/templates/memcheck.rhtml b/report-generators/templates/memcheck.rhtml new file mode 100644 index 000000000..75872ed49 --- /dev/null +++ b/report-generators/templates/memcheck.rhtml @@ -0,0 +1,30 @@ + + + +
Tests passedTests failed
<%= total_passed %>><%= total_failed %>
+ +<% schedules.each do |s| %> +

<%= s.dir.sub('./unit-tests/', '') %>

+ + + +<% s.schedules.each do |t| %> + + + <% if t.status.success? %> + + <% else %> + + <% end %> + + <% stats = extract_stats(t) %> + + + + + +<% end %> +
TestResultDefinitely lostindirectly lostpossibly loststill reachable
+ <%= t.desc %> + passfail<%= stats.definitely_lost %><%= stats.indirectly_lost %><%= stats.possibly_lost %><%= stats.reachable %>
+<% end %> diff --git a/report-generators/templates/unit_detail.rhtml b/report-generators/templates/unit_detail.rhtml new file mode 100644 index 000000000..5324f07cc --- /dev/null +++ b/report-generators/templates/unit_detail.rhtml @@ -0,0 +1,37 @@ + + + + + <% if t.status.success? %> + + <% else %> + + <% end %> + +
TestResult
+ <%= t.desc %> + passfail
+ + + + + + +
Command line
+
+<%= t.command_line %>
+  
+
+ + + + + + + +
Output
+
+<%= t.output %>
+  
+
+ diff --git a/report-generators/templates/unit_test.rhtml b/report-generators/templates/unit_test.rhtml new file mode 100644 index 000000000..3137abdd8 --- /dev/null +++ b/report-generators/templates/unit_test.rhtml @@ -0,0 +1,23 @@ + + + +
Tests passedTests failed
<%= total_passed %>><%= total_failed %>
+ +<% schedules.each do |s| %> +

<%= s.dir.sub('./unit-tests/', '') %>

+ + +<% s.schedules.each do |t| %> + + + <% if t.status.success? %> + + <% else %> + + <% end %> + +<% end %> +
TestResult
+ <%= t.desc %> + passfail
+<% end %> diff --git a/report-generators/test/example.schedule b/report-generators/test/example.schedule new file mode 100644 index 000000000..f617187a1 --- /dev/null +++ b/report-generators/test/example.schedule @@ -0,0 +1,4 @@ +# This is a comment +description number 1:$TEST_TOOL ls +foo bar: $TEST_TOOL du -hs . + this comment is prefixed with whitespace: $TEST_TOOL date \ No newline at end of file diff --git a/report-generators/test/strings/more_strings/test3.txt b/report-generators/test/strings/more_strings/test3.txt new file mode 100644 index 000000000..3e9ffe066 --- /dev/null +++ b/report-generators/test/strings/more_strings/test3.txt @@ -0,0 +1 @@ +lorem diff --git a/report-generators/test/strings/test1.txt b/report-generators/test/strings/test1.txt new file mode 100644 index 000000000..af5626b4a --- /dev/null +++ b/report-generators/test/strings/test1.txt @@ -0,0 +1 @@ +Hello, world! diff --git a/report-generators/test/strings/test2 b/report-generators/test/strings/test2 new file mode 100644 index 000000000..54d55bf0b --- /dev/null +++ b/report-generators/test/strings/test2 @@ -0,0 +1,3 @@ +one +two +three \ No newline at end of file diff --git a/report-generators/test/tc_log.rb b/report-generators/test/tc_log.rb new file mode 100644 index 000000000..a7e9023c2 --- /dev/null +++ b/report-generators/test/tc_log.rb @@ -0,0 +1,26 @@ +require 'test/unit' +require 'stringio' +require 'log' + +class TestLog < Test::Unit::TestCase + include Log + + private + def remove_timestamps(l) + l.gsub(/\[[^\]]*\]/, '') + end + + public + def test_log + StringIO.open do |out| + init(out) + + info("msg1") + warning("msg2") + debug("msg3") + + assert_equal("I, INFO -- : msg1\nW, WARN -- : msg2\nD, DEBUG -- : msg3\n", + remove_timestamps(out.string)) + end + end +end diff --git a/report-generators/test/tc_schedule_file.rb b/report-generators/test/tc_schedule_file.rb new file mode 100644 index 000000000..ec4bc3ea4 --- /dev/null +++ b/report-generators/test/tc_schedule_file.rb @@ -0,0 +1,28 @@ +require 'test/unit' +require 'pathname' +require 'schedule_file' + +class TestScheduleFile < Test::Unit::TestCase + def test_reading + p = Pathname.new("report-generators/test/example.schedule") + p.open do |f| + s = Schedule.read(p.dirname, f) + + assert_equal(3, s.schedules.size) + assert_equal(s.schedules[2].desc, "this comment is prefixed with whitespace") + assert_equal(s.schedules[0].command_line, "$TEST_TOOL ls") + end + end + + def test_running + p = Pathname.new("report-generators/test/example.schedule") + p.open do |f| + s = Schedule.read(p.dirname, f) + s.run + + s.schedules.each do |t| + assert(t.status.success?) + end + end + end +end diff --git a/report-generators/test/tc_string_store.rb b/report-generators/test/tc_string_store.rb new file mode 100644 index 000000000..5ff89dd8a --- /dev/null +++ b/report-generators/test/tc_string_store.rb @@ -0,0 +1,19 @@ +require 'string-store' +require 'test/unit' + +class TestStringStore < Test::Unit::TestCase + def setup + @ss = StringStore.new(['report-generators/test/strings', + 'report-generators/test/strings/more_strings']) + end + + def test_lookup + assert_equal("Hello, world!\n", @ss.lookup(:test1)) + assert_equal("one\ntwo\nthree", @ss.lookup(:test2)) + assert_equal("lorem\n", @ss.lookup(:test3)) + + assert_raises(RuntimeError) do + @ss.lookup(:unlikely_name) + end + end +end diff --git a/report-generators/test/ts.rb b/report-generators/test/ts.rb new file mode 100644 index 000000000..0a8cc9102 --- /dev/null +++ b/report-generators/test/ts.rb @@ -0,0 +1,3 @@ +require 'tc_log' +require 'tc_string_store' +require 'tc_schedule_file' diff --git a/report-generators/title_page.rb b/report-generators/title_page.rb new file mode 100644 index 000000000..f6873eb17 --- /dev/null +++ b/report-generators/title_page.rb @@ -0,0 +1,32 @@ +# This generates the index for the reports, including generation +# times. + +require 'log' +require 'string-store' +require 'reports' +require 'erb' +require 'report_templates' + +include Reports + +reports = ReportRegister.new + +def safe_mtime(r) + r.path.file? ? r.path.mtime.to_s : "not generated" +end + +template_store = TemplateStringStore.new + +# FIXME: use generate_report() method +erb = ERB.new(template_store.lookup("index.rhtml")) +body = erb.result(binding) +title = "Generation times" + +erb = ERB.new(template_store.lookup("boiler_plate.rhtml")) +txt = erb.result(binding) + +Pathname.new("reports/index.html").open("w") do |f| + f.puts txt +end + + diff --git a/report-generators/unit_test.rb b/report-generators/unit_test.rb new file mode 100644 index 000000000..ebc378550 --- /dev/null +++ b/report-generators/unit_test.rb @@ -0,0 +1,46 @@ +# Reads the schedule files given on the command line. Runs them and +# generates the reports. + +require 'schedule_file' +require 'pathname' +require 'reports' +require 'erb' +require 'report_templates' + +include ReportTemplates + +schedules = ARGV.map do |f| + p = Pathname.new(f) + Schedule.read(p.dirname, p) +end + +total_passed = 0 +total_failed = 0 + +# We need to make sure the lvm shared libs are in the LD_LIBRARY_PATH +ENV['LD_LIBRARY_PATH'] = `pwd`.strip + "/libdm:" + (ENV['LD_LIBRARY_PATH'] || '') + +schedules.each do |s| + s.run + + s.schedules.each do |t| + if t.status.success? + total_passed += 1 + else + total_failed += 1 + end + end +end + +def mangle(txt) + txt.gsub(/\s+/, '_') +end + +generate_report(:unit_test, binding) + +# now we generate a detail report for each schedule +schedules.each do |s| + s.schedules.each do |t| + generate_report(:unit_detail, binding, Pathname.new("reports/detail_#{mangle(t.desc)}.html")) + end +end diff --git a/reports/stylesheet.css b/reports/stylesheet.css new file mode 100644 index 000000000..136baf691 --- /dev/null +++ b/reports/stylesheet.css @@ -0,0 +1,264 @@ +/* I know nothing about css, and am not likely too either since it's + * about bottom on my list of things to learn. So this file has been + * pinched from the Pragmatic Programmers ruby on rails book. - ejt + */ + +/* Global styles */ + +/* START:notice */ +#notice { + border: 2px solid red; + padding: 1em; + margin-bottom: 2em; + background-color: #f0f0f0; + font: bold smaller sans-serif; +} +/* END:notice */ + +/* Styles for admin/list */ + +#product-list .list-title { + color: #244; + font-weight: bold; + font-size: larger; +} + +#product-list .list-image { + width: 60px; + height: 70px; +} + + +#product-list .list-actions { + font-size: x-small; + text-align: right; + padding-left: 1em; +} + +#product-list .list-line-even { + background: #e0f8f8; +} + +#product-list .list-line-odd { + background: #f8b0f8; +} + + +/* Styles for main page */ + +#banner { + background: #9c9; + padding-top: 5px; + padding-bottom: 5px; + border-bottom: 2px solid; + font: small-caps 20px/20px "Times New Roman", serif; + color: #282; + text-align: center; +} + +#banner img { + float: left; +} + +#main { + margin-left: 0em; + padding-top: 4ex; + padding-left: 2em; + background: white; +} + +h1 { + font: 150% sans-serif; + color: #226; + border-bottom: 3px dotted #77d; +} + +/* An entry in the store catalog */ + +#store .entry { + border-bottom: 1px dotted #77d; +} + +#store .title { + font-size: 120%; + font-family: sans-serif; +} + +#store .entry img { + width: 75px; + float: left; +} + + +#store .entry h3 { + margin-bottom: 2px; + color: #227; +} + +#store .entry p { + margin-top: 0px; + margin-bottom: 0.8em; +} + +#store .entry .price-line { +} + +#store .entry .add-to-cart { + position: relative; +} + +#store .entry .price { + color: #44a; + font-weight: bold; + margin-right: 2em; +} + +/* START:inline */ +#store .entry form, #store .entry form div { + display: inline; +} +/* END:inline */ + +/* START:cart */ +/* Styles for the cart in the main page and the sidebar */ + +.cart-title { + font: 120% bold; +} + +.item-price, .total-line { + text-align: right; +} + +.total-line .total-cell { + font-weight: bold; + border-top: 1px solid #595; +} + + +/* Styles for the cart in the sidebar */ + +#cart, #cart table { + font-size: smaller; + color: white; +} + +#cart table { + border-top: 1px dotted #595; + border-bottom: 1px dotted #595; + margin-bottom: 10px; +} +/* END:cart */ + +/* Styles for order form */ + +.depot-form fieldset { + background: #efe; +} + +.depot-form legend { + color: #dfd; + background: #141; + font-family: sans-serif; + padding: 0.2em 1em; +} + +.depot-form label { + width: 5em; + float: left; + text-align: right; + margin-right: 0.5em; + display: block; +} + +.depot-form .submit { + margin-left: 5.5em; +} + +/* The error box */ + +.fieldWithErrors { + padding: 2px; + background-color: red; + display: table; +} + +#errorExplanation { + width: 400px; + border: 2px solid red; + padding: 7px; + padding-bottom: 12px; + margin-bottom: 20px; + background-color: #f0f0f0; +} + +#errorExplanation h2 { + text-align: left; + font-weight: bold; + padding: 5px 5px 5px 15px; + font-size: 12px; + margin: -7px; + background-color: #c00; + color: #fff; +} + +#errorExplanation p { + color: #333; + margin-bottom: 0; + padding: 5px; +} + +#errorExplanation ul li { + font-size: 12px; + list-style: square; +} + +body { + font: normal 75% verdana,arial,helvetica; + color:#000000; +} + +table tr td, table tr th { + font-size: 75%; +} + +table.stripes tr th { + font-weight: bold; + text-align: left; + background: #a0a0a0; +} + +table.stripes tr td { + background: #ccccc0; +} + +td.pass { + color: green; +} + +td.fail { + color: red; + font-weight: bold; +} + +#main { + padding-left: 0em; +} + +#controls { + float: left; + padding-top: 1em; + padding-left: 1em; + padding-right: 1em; + padding-bottom: 1em; + width: 14em; + border-right: 2px solid; +} + +#body { + margin-left: 16em; + padding-top: 4ex; + padding-left: 2em; + background: white; + border-left: 2px solid; +}