[poppler] 2 commits - regtest/Printer.py regtest/TestReferences.py regtest/TestRun.py
Carlos Garcia Campos
carlosgc at kemper.freedesktop.org
Wed Nov 27 09:51:12 PST 2013
regtest/Printer.py | 44 ++----------
regtest/TestReferences.py | 50 +++++++++-----
regtest/TestRun.py | 164 +++++++++++++++++++++++++---------------------
3 files changed, 135 insertions(+), 123 deletions(-)
New commits:
commit 1486d93f0f6418b37ccc57568717d9349a79935b
Author: Carlos Garcia Campos <carlosgc at gnome.org>
Date: Wed Nov 27 18:29:36 2013 +0100
regtest: Do not consider docs with no refs as skipped
Handle them differently as new docs, and show them in the summary
suggesting to use create-refs command to include them in the test
suite.
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
index f47485f..23ff31f 100644
--- a/regtest/TestRun.py
+++ b/regtest/TestRun.py
@@ -47,6 +47,7 @@ class TestRun:
self._failed_status_error = []
self._stderr = []
self._skipped = []
+ self._new = []
self._queue = Queue()
self._lock = RLock()
@@ -72,7 +73,7 @@ class TestRun:
ref_is_failed = backend.is_failed(refs_path)
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
with self._lock:
- self._skipped.append("%s (%s)" % (doc_path, backend.get_name()))
+ self._new.append("%s (%s)" % (doc_path, backend.get_name()))
self._n_tests += 1
self.printer.print_default("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
return
@@ -153,7 +154,7 @@ class TestRun:
if not os.path.isdir(refs_path):
with self._lock:
- self._skipped.append("%s" % (doc_path))
+ self._new.append("%s" % (doc_path))
self._n_tests += len(backends)
self.printer.print_default("Reference dir not found for %s, skipping" % (doc_path))
return
@@ -190,24 +191,29 @@ class TestRun:
self._queue.join()
def summary(self):
- if not self._n_run:
- self.printer.printout_ln("No tests run")
- return
-
- self.printer.printout_ln()
- self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run))
self.printer.printout_ln()
- def report_tests(test_list, test_type):
- n_tests = len(test_list)
- if not n_tests:
- return
- self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_run, ", ".join(test_list)))
+
+ if self._n_run:
+ self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run))
self.printer.printout_ln()
+ def report_tests(test_list, test_type):
+ n_tests = len(test_list)
+ if not n_tests:
+ return
+ self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_run, ", ".join(test_list)))
+ self.printer.printout_ln()
+
+ report_tests(self._failed, "failed")
+ report_tests(self._crashed, "crashed")
+ report_tests(self._failed_status_error, "failed to run")
+ report_tests(self._stderr, "have stderr output")
+ else:
+ self.printer.printout_ln("No tests run")
- report_tests(self._failed, "failed")
- report_tests(self._crashed, "crashed")
- report_tests(self._failed_status_error, "failed to run")
- report_tests(self._stderr, "have stderr output")
if self._skipped:
self.printer.printout_ln("%d tests skipped: %s" % (len(self._skipped), ", ".join(self._skipped)))
self.printer.printout_ln()
+
+ if self._new:
+ self.printer.printout_ln("%d new documents: %s\nUse create-refs command to add reference results for them" % (len(self._new), ", ".join(self._new)))
+ self.printer.printout_ln()
commit 5241c0cc730c380dc44c3a081e1de675e1915861
Author: Carlos Garcia Campos <carlosgc at gnome.org>
Date: Wed Nov 27 17:18:37 2013 +0100
regtest: Use number of tests run to show the progress
Instead of using the number of document tested. We don't really need to
set a number to a particular document, we only want to know the progress
of the whole process. This ensures that the progress is shown in the
correct order. It also simplifies the code a bit.
Also improved the output formatting to make it easier to read.
diff --git a/regtest/Printer.py b/regtest/Printer.py
index 0b0b34a..23dfd34 100644
--- a/regtest/Printer.py
+++ b/regtest/Printer.py
@@ -34,14 +34,12 @@ class Printer:
self._rewrite = self._stream.isatty() and not self._verbose
self._current_line = None
- self._tests = {}
-
self._lock = RLock()
Printer.__single = self
def _erase_current_line(self):
- if not self._rewrite or self._current_line is None:
+ if self._current_line is None:
return
line_len = len(self._current_line)
@@ -58,25 +56,17 @@ class Printer:
self._stream.flush()
def printout(self, msg):
+ if not self._rewrite:
+ self.printout_ln(msg)
+
with self._lock:
self._erase_current_line()
self._print(msg)
self._current_line = msg[msg.rfind('\n') + 1:]
- def printout_update(self, msg):
+ def printout_ln(self, msg=''):
with self._lock:
- if self._rewrite and self._current_line is not None:
- msg = self._current_line + msg
- elif not self._rewrite:
- msg = self._ensure_new_line(msg)
- self.printout(msg)
-
- def printout_ln(self, msg):
- with self._lock:
- if self._current_line is not None:
- self._current_line = None
- msg = '\n' + msg
-
+ self._erase_current_line()
self._print(self._ensure_new_line(msg))
def printerr(self, msg):
@@ -84,25 +74,11 @@ class Printer:
self.stderr.write(self._ensure_new_line(msg))
self.stderr.flush()
- def print_test_start(self, doc_path, backend_name, n_doc, total_docs):
- with self._lock:
- self._tests[(doc_path, backend_name)] = n_doc, total_docs
-
- def print_test_result(self, doc_path, backend_name, msg):
- if not self._rewrite:
- self.print_test_result_ln(doc_path, backend_name, msg)
- return
+ def print_test_result(self, doc_path, backend_name, n_test, total_tests, msg):
+ self.printout("[%d/%d] %s (%s): %s" % (n_test, total_tests, doc_path, backend_name, msg))
- with self._lock:
- n_doc, total_docs = self._tests.pop((doc_path, backend_name))
- msg = "Tested '%s' using %s backend (%d/%d): %s" % (doc_path, backend_name, n_doc, total_docs, msg)
- self.printout(msg)
-
- def print_test_result_ln(self, doc_path, backend_name, msg):
- with self._lock:
- n_doc, total_docs = self._tests.pop((doc_path, backend_name))
- msg = "Tested '%s' using %s backend (%d/%d): %s" % (doc_path, backend_name, n_doc, total_docs, msg)
- self.printout_ln(msg)
+ def print_test_result_ln(self, doc_path, backend_name, n_test, total_tests, msg):
+ self.printout_ln("[%d/%d] %s (%s): %s" % (n_test, total_tests, doc_path, backend_name, msg))
def print_default(self, msg):
if self._verbose:
diff --git a/regtest/TestReferences.py b/regtest/TestReferences.py
index 6cedb4b..4572ef6 100644
--- a/regtest/TestReferences.py
+++ b/regtest/TestReferences.py
@@ -24,7 +24,7 @@ from Printer import get_printer
from Utils import get_document_paths_from_dir, get_skipped_tests
from Queue import Queue
-from threading import Thread
+from threading import Thread, RLock
class TestReferences:
@@ -34,8 +34,11 @@ class TestReferences:
self._skipped = get_skipped_tests(docsdir)
self.config = Config()
self.printer = get_printer()
+ self._total_tests = 1
+ self._n_tests = 0
self._queue = Queue()
+ self._lock = RLock()
try:
os.makedirs(self._refsdir)
@@ -45,9 +48,19 @@ class TestReferences:
except:
raise
- def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1):
+ def _get_backends(self):
+ if self.config.backends:
+ return [get_backend(name) for name in self.config.backends]
+
+ return get_all_backends()
+
+ def create_refs_for_file(self, filename):
+ backends = self._get_backends()
+
if filename in self._skipped:
- self.printer.print_default("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
+ with self._lock:
+ self._n_tests += len(backends)
+ self.printer.print_default("Skipping test '%s'" % (os.path.join(self._docsdir, filename)))
return
refs_path = os.path.join(self._refsdir, filename)
@@ -60,38 +73,43 @@ class TestReferences:
raise
doc_path = os.path.join(self._docsdir, filename)
- if self.config.backends:
- backends = [get_backend(name) for name in self.config.backends]
- else:
- backends = get_all_backends()
-
for backend in backends:
if not self.config.force and backend.has_results(refs_path):
- self.printer.print_default("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
+ with self._lock:
+ self._n_tests += 1
+ self.printer.print_default("Results found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
continue
- self.printer.printout_ln("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
+
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
+ with self._lock:
+ self._n_tests += 1
+ self.printer.printout_ln("[%d/%d] %s (%s): done" % (self._n_tests, self._total_tests, doc_path, backend.get_name()))
def _worker_thread(self):
while True:
- doc, n_doc, total_docs = self._queue.get()
- self.create_refs_for_file(doc, n_doc, total_docs)
+ doc = self._queue.get()
+ self.create_refs_for_file(doc)
self._queue.task_done()
def create_refs(self):
docs, total_docs = get_document_paths_from_dir(self._docsdir)
+ backends = self._get_backends()
+ self._total_tests = total_docs * len(backends)
+
+ self.printer.printout_ln('Found %d documents' % (total_docs))
+ self.printer.printout_ln('Backends: %s' % ', '.join([backend.get_name() for backend in backends]))
+ self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), self.config.threads))
+ self.printer.printout_ln()
- self.printer.printout_ln('Process %d is spawning %d worker threads...' % (os.getpid(), self.config.threads))
+ self.printer.printout('Spawning %d workers...' % (self.config.threads))
for n_thread in range(self.config.threads):
thread = Thread(target=self._worker_thread)
thread.daemon = True
thread.start()
- n_doc = 0
for doc in docs:
- n_doc += 1
- self._queue.put( (doc, n_doc, total_docs) )
+ self._queue.put(doc)
self._queue.join()
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
index 8f8513c..f47485f 100644
--- a/regtest/TestRun.py
+++ b/regtest/TestRun.py
@@ -36,9 +36,11 @@ class TestRun:
self._skip = get_skipped_tests(docsdir)
self.config = Config()
self.printer = get_printer()
+ self._total_tests = 1
# Results
self._n_tests = 0
+ self._n_run = 0
self._n_passed = 0
self._failed = []
self._crashed = []
@@ -57,7 +59,13 @@ class TestRun:
except:
raise
- def test(self, refs_path, doc_path, test_path, backend, n_doc, total_docs):
+ def _get_backends(self):
+ if self.config.backends:
+ return [get_backend(name) for name in self.config.backends]
+
+ return get_all_backends()
+
+ def test(self, refs_path, doc_path, test_path, backend):
# First check whether there are test results for the backend
ref_has_md5 = backend.has_md5(refs_path)
ref_is_crashed = backend.is_crashed(refs_path)
@@ -65,70 +73,71 @@ class TestRun:
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
with self._lock:
self._skipped.append("%s (%s)" % (doc_path, backend.get_name()))
+ self._n_tests += 1
self.printer.print_default("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
return
+ test_has_md5 = backend.create_refs(doc_path, test_path)
+ test_passed = False
+ if ref_has_md5 and test_has_md5:
+ test_passed = backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs)
+
with self._lock:
self._n_tests += 1
+ self._n_run += 1
- self.printer.print_test_start(doc_path, backend.get_name(), n_doc, total_docs)
- test_has_md5 = backend.create_refs(doc_path, test_path)
-
- if backend.has_stderr(test_path):
- with self._lock:
+ if backend.has_stderr(test_path):
self._stderr.append("%s (%s)" % (doc_path, backend.get_name()))
- if ref_has_md5 and test_has_md5:
- if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs):
- # FIXME: remove dir if it's empty?
- self.printer.print_test_result(doc_path, backend.get_name(), "PASS")
- with self._lock:
+ if ref_has_md5 and test_has_md5:
+ if test_passed:
+ # FIXME: remove dir if it's empty?
+ self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS")
self._n_passed += 1
- else:
- self.printer.print_test_result_ln(doc_path, backend.get_name(), "FAIL")
- with self._lock:
+ else:
+ self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "FAIL")
self._failed.append("%s (%s)" % (doc_path, backend.get_name()))
- return
- elif test_has_md5:
- if ref_is_crashed:
- self.printer.print_test_result_ln(doc_path, backend.get_name(), "DOES NOT CRASH")
- elif ref_is_failed:
- self.printer.print_test_result_ln(doc_path, backend.get_name(), "DOES NOT FAIL")
- return
+ return
- test_is_crashed = backend.is_crashed(test_path)
- if ref_is_crashed and test_is_crashed:
- self.printer.print_test_result(doc_path, backend.get_name(), "PASS (Expected crash)")
- with self._lock:
+ if test_has_md5:
+ if ref_is_crashed:
+ self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "DOES NOT CRASH")
+ elif ref_is_failed:
+ self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "DOES NOT FAIL")
+ return
+
+ test_is_crashed = backend.is_crashed(test_path)
+ if ref_is_crashed and test_is_crashed:
+ self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS (Expected crash)")
self._n_passed += 1
- return
+ return
- test_is_failed = backend.is_failed(test_path)
- if ref_is_failed and test_is_failed:
- # FIXME: compare status errors
- self.printer.print_test_result(doc_path, backend.get_name(), "PASS (Expected fail with status error %d)" % (test_is_failed))
- with self._lock:
+ test_is_failed = backend.is_failed(test_path)
+ if ref_is_failed and test_is_failed:
+ # FIXME: compare status errors
+ self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS (Expected fail with status error %d)" % (test_is_failed))
self._n_passed += 1
- return
+ return
- if test_is_crashed:
- self.printer.print_test_result_ln(doc_path, backend.get_name(), "CRASH")
- with self._lock:
+ if test_is_crashed:
+ self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "CRASH")
self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
- return
+ return
- if test_is_failed:
- self.printer.print_test_result_ln(doc_path, backend.get_name(), "FAIL (status error %d)" % (test_is_failed))
- with self._lock:
+ if test_is_failed:
+ self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "FAIL (status error %d)" % (test_is_failed))
self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
- return
+ return
+
+ def run_test(self, filename):
+ backends = self._get_backends()
- def run_test(self, filename, n_doc = 1, total_docs = 1):
if filename in self._skip:
doc_path = os.path.join(self._docsdir, filename)
with self._lock:
self._skipped.append("%s" % (doc_path))
- self.printer.print_default("Skipping test '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
+ self._n_tests += len(backends)
+ self.printer.print_default("Skipping test '%s'" % (doc_path))
return
out_path = os.path.join(self._outdir, filename)
@@ -145,57 +154,60 @@ class TestRun:
if not os.path.isdir(refs_path):
with self._lock:
self._skipped.append("%s" % (doc_path))
- self.printer.print_default("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
+ self._n_tests += len(backends)
+ self.printer.print_default("Reference dir not found for %s, skipping" % (doc_path))
return
- if self.config.backends:
- backends = [get_backend(name) for name in self.config.backends]
- else:
- backends = get_all_backends()
-
for backend in backends:
- self.test(refs_path, doc_path, out_path, backend, n_doc, total_docs)
+ self.test(refs_path, doc_path, out_path, backend)
def _worker_thread(self):
while True:
- doc, n_doc, total_docs = self._queue.get()
- self.run_test(doc, n_doc, total_docs)
+ doc = self._queue.get()
+ self.run_test(doc)
self._queue.task_done()
def run_tests(self):
docs, total_docs = get_document_paths_from_dir(self._docsdir)
+ backends = self._get_backends()
+ self._total_tests = total_docs * len(backends)
+
+ self.printer.printout_ln('Found %d documents' % (total_docs))
+ self.printer.printout_ln('Backends: %s' % ', '.join([backend.get_name() for backend in backends]))
+ self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), self.config.threads))
+ self.printer.printout_ln()
- self.printer.printout_ln('Process %d is spawning %d worker threads...' % (os.getpid(), self.config.threads))
+ self.printer.printout('Spawning %d workers...' % (self.config.threads))
for n_thread in range(self.config.threads):
thread = Thread(target=self._worker_thread)
thread.daemon = True
thread.start()
- n_doc = 0
for doc in docs:
- n_doc += 1
- self._queue.put( (doc, n_doc, total_docs) )
+ self._queue.put(doc)
self._queue.join()
def summary(self):
- if not self._n_tests:
+ if not self._n_run:
self.printer.printout_ln("No tests run")
return
- self.printer.printout_ln("Total %d tests" % (self._n_tests))
- self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
+ self.printer.printout_ln()
+ self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run))
+ self.printer.printout_ln()
def report_tests(test_list, test_type):
n_tests = len(test_list)
if not n_tests:
return
- self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
+ self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_run, ", ".join(test_list)))
+ self.printer.printout_ln()
report_tests(self._failed, "failed")
report_tests(self._crashed, "crashed")
report_tests(self._failed_status_error, "failed to run")
report_tests(self._stderr, "have stderr output")
- report_tests(self._skipped, "skipped")
-
-
+ if self._skipped:
+ self.printer.printout_ln("%d tests skipped: %s" % (len(self._skipped), ", ".join(self._skipped)))
+ self.printer.printout_ln()
More information about the poppler
mailing list