[poppler] 2 commits - regtest/backends regtest/commands regtest/HTMLReport.py
Carlos Garcia Campos
carlosgc at kemper.freedesktop.org
Sat Feb 25 12:02:38 PST 2012
regtest/HTMLReport.py | 197 ++++++++++++++++++++++++++++++++++++++
regtest/backends/__init__.py | 36 ++++++
regtest/backends/cairo.py | 2
regtest/backends/splash.py | 2
regtest/backends/text.py | 2
regtest/commands/create-report.py | 57 ++++++++++
6 files changed, 290 insertions(+), 6 deletions(-)
New commits:
commit 738b879ebb536cc84d7ec96543d484023b69e6d3
Author: Carlos Garcia Campos <carlosgc at gnome.org>
Date: Sat Feb 25 20:53:58 2012 +0100
regtest: Add create-report command to generate html report of test results
diff --git a/regtest/HTMLReport.py b/regtest/HTMLReport.py
new file mode 100644
index 0000000..f64ec2d
--- /dev/null
+++ b/regtest/HTMLReport.py
@@ -0,0 +1,197 @@
+# HTMLReport.py
+#
+# Copyright (C) 2012 Carlos Garcia Campos <carlosgc at gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from backends import get_backend, get_all_backends
+from Config import Config
+import os
+import subprocess
+
+class BackendTestResult:
+
+ def __init__(self, test, refsdir, outdir, backend, results):
+ self._test = test
+ self._refsdir = refsdir
+ self._outdir = outdir
+ self._backend = backend
+
+ self._results = []
+
+ ref_path = os.path.join(self._refsdir, self._test)
+ ref_names = backend.get_ref_names(ref_path)
+ for result in results:
+ basename = os.path.basename(result)
+ if basename in ref_names:
+ self._results.append(basename)
+
+ def is_failed(self):
+ return len(self._results) > 0
+
+ def is_crashed(self):
+ return self._backend.is_crashed(os.path.join(self._outdir, self._test))
+
+ def is_failed_to_run(self):
+ return self._backend.is_failed(os.path.join(self._outdir, self._test))
+
+ def get_stderr(self):
+ return self._backend.get_stderr(os.path.join(self._outdir, self._test))
+
+ def get_failed_html(self):
+ html = ""
+ for result in self._results:
+ actual = os.path.abspath(os.path.join(self._outdir, self._test, result))
+ expected = os.path.abspath(os.path.join(self._refsdir, self._test, result))
+ html += "<li><a href='%s'>actual</a> <a href='%s'>expected</a> " % (actual, expected)
+ if self._backend.has_diff(actual):
+ html += "<a href='%s'>diff</a>" % (os.path.abspath(actual + self._backend.get_diff_ext()))
+ html += "</li>\n"
+
+ if html:
+ return "<ul>%s</ul>\n" % (html)
+ return ""
+
+
+class TestResult:
+
+ def __init__(self, docsdir, refsdir, outdir, resultdir, results, backends):
+ self._refsdir = refsdir
+ self._outdir = outdir
+
+ self._test = resultdir[len(self._outdir):]
+ self._doc = os.path.join(docsdir, self._test)
+ self._results = {}
+ for backend in backends:
+ self._results[backend] = BackendTestResult(self._test, refsdir, outdir, backend, results)
+
+ def get_test(self):
+ return self._test
+
+ def is_failed(self):
+ for backend in self._results:
+ if self._results[backend].is_failed():
+ return True
+ return False;
+
+ def get_failed_html(self):
+ html = ""
+ for backend in self._results:
+ backend_html = self._results[backend].get_failed_html()
+ if not backend_html:
+ continue
+
+ html += "<li>%s " % (backend.get_name())
+ stderr = self._results[backend].get_stderr()
+ if os.path.exists(stderr):
+ html += "<a href='%s'>stderr</a>" % (os.path.abspath(stderr))
+ html += "</li>\n%s" % (backend_html)
+
+ if html:
+ return "<h2><a name='%s'><a href='%s'>%s</a></a></h2>\n<ul>%s</ul><a href='#top'>Top</a>\n" % (self._test, self._doc, self._test, html)
+ return ""
+
+ def get_crashed_html(self):
+ html = ""
+ for backend in self._results:
+ if not self._results[backend].is_crashed():
+ continue
+
+ html += "<li><a href='%s'>%s</a> (%s)</li>\n" % (self._doc, self._test, backend.get_name())
+
+ if html:
+ return "<ul>%s</ul>\n" % (html)
+ return ""
+
+ def get_failed_to_run_html(self):
+ html = ""
+ for backend in self._results:
+ status = self._results[backend].is_failed_to_run()
+ if not status:
+ continue
+
+ html += "<li><a href='%s'>%s</a> [Status: %d] (%s)</li>\n" % (self._doc, self._test, status, backend.get_name())
+
+ if html:
+ return "<ul>%s</ul>\n" % (html)
+ return ""
+
+class HTMLReport:
+
+ def __init__(self, docsdir, refsdir, outdir):
+ self._docsdir = docsdir
+ self._refsdir = refsdir
+ self._outdir = outdir
+ self.config = Config()
+
+ def create(self):
+ html = "<html><body><a name='top'></a>"
+ if self.config.backends:
+ backends = [get_backend(name) for name in self.config.backends]
+ else:
+ backends = get_all_backends()
+
+ results = {}
+ for root, dirs, files in os.walk(self._outdir, False):
+ if not files:
+ continue
+ if not root.lower().endswith('.pdf'):
+ continue
+
+ results[root] = TestResult(self._docsdir, self._refsdir, self._outdir, root, files, backends)
+
+ tests = results.keys()
+ tests.sort()
+
+ failed_anchors = []
+ failed = ""
+ crashed = ""
+ failed_to_run = ""
+ for test_name in tests:
+ test = results[test_name]
+ if test.is_failed():
+ failed_anchors.append(test.get_test())
+ failed += test.get_failed_html()
+ crashed += test.get_crashed_html()
+ failed_to_run += test.get_failed_to_run_html()
+
+ if failed:
+ failed = "<h1><a name='failed'>Tests Failed (differences were found)</name></h1>\n%s" % (failed)
+ if crashed:
+ crashed = "<h1><a name='crashed'>Tests Crashed</a></h1>\n%s" % (crashed)
+ if failed_to_run:
+ failed_to_run = "<h1><a name='failed_to_run'>Tests that failed to run (command returned an error status)</a></h1>\n%s" % (failed_to_run)
+
+ if failed or crashed or failed_to_run:
+ html += "<ul>\n"
+ if failed:
+ html += "<li><a href='#failed'>Tests Failed (differences were found)</a></li>\n<ul>"
+ for anchor in failed_anchors:
+ html += "<li><a href='#%s'>%s</a></li>" % (anchor, anchor)
+ html += "</ul>\n"
+ if crashed:
+ html += "<li><a href='#crashed'>Tests Crashed(differences were found)</a></li>\n"
+ if failed_to_run:
+ html += "<li><a href='#failed_to_run'>Tests that failed to run (command returned an error status)</a></li>\n"
+ html += "</ul>\n"
+
+ html += failed + crashed + failed_to_run + "</body></html>"
+
+ report_index = os.path.join(self._outdir, 'index.html')
+ f = open(report_index, 'wb')
+ f.write(html)
+ f.close()
+
+ subprocess.Popen(['xdg-open', report_index])
diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
index 77e5498..26be0b0 100644
--- a/regtest/backends/__init__.py
+++ b/regtest/backends/__init__.py
@@ -136,6 +136,21 @@ class Backend:
return retval
+ def get_ref_names(self, refs_path):
+ retval = []
+ md5_path = os.path.join(refs_path, self._name)
+ md5_file = open(md5_path + '.md5', 'r')
+ for line in md5_file.readlines():
+ md5sum, ref_path = line.strip('\n').split(' ', 1)
+ basename = os.path.basename(ref_path)
+ if not self.__should_have_checksum(basename):
+ continue
+
+ retval.append(basename)
+ md5_file.close()
+
+ return retval
+
def has_md5(self, test_path):
return os.path.exists(os.path.join(test_path, self._name + '.md5'))
@@ -156,8 +171,11 @@ class Backend:
def has_results(self, test_path):
return self.has_md5(test_path) or self.is_crashed(test_path) or self.is_failed(test_path)
+ def get_stderr(self, test_path):
+ return os.path.join(test_path, self._name + '.stderr')
+
def has_stderr(self, test_path):
- return os.path.exists(os.path.join(test_path, self._name + '.stderr'))
+ return os.path.exists(self.get_stderr(test_path))
def has_diff(self, test_result):
if not self._diff_ext:
diff --git a/regtest/commands/create-report.py b/regtest/commands/create-report.py
new file mode 100644
index 0000000..89b6687
--- /dev/null
+++ b/regtest/commands/create-report.py
@@ -0,0 +1,57 @@
+# create-report.py
+#
+# Copyright (C) 2012 Carlos Garcia Campos <carlosgc at gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from commands import Command, register_command
+from HTMLReport import HTMLReport
+from Config import Config
+import os
+import tempfile
+
+class CreateReport(Command):
+
+ name = 'create-report'
+ usage_args = '[ options ... ] tests '
+ description = 'Create report of test results'
+
+ def __init__(self):
+ Command.__init__(self)
+ parser = self._get_args_parser()
+ parser.add_argument('--refs-dir',
+ action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'),
+ help = 'Directory containing the references')
+ parser.add_argument('-o', '--out-dir',
+ action = 'store', dest = 'out_dir', default = os.path.join(tempfile.gettempdir(), 'out'),
+ help = 'Directory containing the results')
+ parser.add_argument('tests')
+
+ def run(self, options):
+ config = Config()
+
+ doc = options['tests']
+ if os.path.isdir(doc):
+ docs_dir = doc
+ else:
+ docs_dir = os.path.dirname(doc)
+
+ report = HTMLReport(docs_dir, options['refs_dir'], options['out_dir'])
+ if doc == docs_dir:
+ report.create()
+ else:
+ report.create_for_file(os.path.basename(doc))
+
+register_command('create-report', CreateReport)
commit d5faabd509c2860ab199ee89b8ef9d4c14fa5118
Author: Carlos Garcia Campos <carlosgc at gnome.org>
Date: Sat Feb 25 20:50:53 2012 +0100
regtest: Use diff.png extension for image diff files
So that they are recognized as images
diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
index 47985fb..77e5498 100644
--- a/regtest/backends/__init__.py
+++ b/regtest/backends/__init__.py
@@ -33,13 +33,17 @@ class UnknownBackendError(Exception):
class Backend:
- def __init__(self, name):
+ def __init__(self, name, diff_ext = None):
self._name = name
+ self._diff_ext = diff_ext
self._utilsdir = Config().utils_dir
def get_name(self):
return self._name
+ def get_diff_ext(self):
+ return self._diff_ext
+
def __should_have_checksum(self, entry):
if not entry.startswith(self._name):
return False
@@ -155,6 +159,14 @@ class Backend:
def has_stderr(self, test_path):
return os.path.exists(os.path.join(test_path, self._name + '.stderr'))
+ def has_diff(self, test_result):
+ if not self._diff_ext:
+ return False
+ basename = os.path.basename(test_result)
+ if not basename.startswith(self._name):
+ return False
+ return os.path.exists(test_result + self._diff_ext)
+
def __create_stderr_file(self, stderr, out_path):
if not stderr:
return
@@ -216,7 +228,7 @@ class Backend:
ref = Image.open(ref_path)
result = Image.open(result_path)
diff = ImageChops.difference(ref, result)
- diff.save(result_path + '.diff', 'png')
+ diff.save(result_path + '.diff.png', 'png')
def _create_diff(self, ref_path, result_path):
raise NotImplementedError
diff --git a/regtest/backends/cairo.py b/regtest/backends/cairo.py
index a990ddf..304783e 100644
--- a/regtest/backends/cairo.py
+++ b/regtest/backends/cairo.py
@@ -23,7 +23,7 @@ import os
class Cairo(Backend):
def __init__(self, name):
- Backend.__init__(self, name)
+ Backend.__init__(self, name, '.diff.png')
self._pdftocairo = os.path.join(self._utilsdir, 'pdftocairo');
def create_refs(self, doc_path, refs_path):
diff --git a/regtest/backends/splash.py b/regtest/backends/splash.py
index 5fbfba6..3144bc7 100644
--- a/regtest/backends/splash.py
+++ b/regtest/backends/splash.py
@@ -23,7 +23,7 @@ import os
class Splash(Backend):
def __init__(self, name):
- Backend.__init__(self, name)
+ Backend.__init__(self, name, '.diff.png')
self._pdftoppm = os.path.join(self._utilsdir, 'pdftoppm');
def create_refs(self, doc_path, refs_path):
diff --git a/regtest/backends/text.py b/regtest/backends/text.py
index 14e7c03..10b660a 100644
--- a/regtest/backends/text.py
+++ b/regtest/backends/text.py
@@ -23,7 +23,7 @@ import os
class Text(Backend):
def __init__(self, name):
- Backend.__init__(self, name)
+ Backend.__init__(self, name, '.diff')
self._pdftotext = os.path.join(self._utilsdir, 'pdftotext');
def create_refs(self, doc_path, refs_path):
More information about the poppler
mailing list