{SECOND_NESTED}
FAIL! -- 2 Failed
"""
# A job that died before any test ran carries neither layout's markers.
UNMARKED_LOG = f"""{STYLE}
[+0004s] make: *** [Makefile:1: binaries] Error 2
"""
class TestCase(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.addCleanup(self.tmpdir.cleanup)
def write_log(self, name, content):
path = os.path.join(self.tmpdir.name, name)
with open(path, 'w') as log:
log.write(content)
return path
def summarize(self, *args):
return subprocess.run([sys.executable, SCRIPT, *args],
capture_output=True, text=True)
def test_ginkgo_log_not_named_int(self):
"""bindings is a ginkgo suite too, its failures must be reported"""
log = self.write_log('bindings-root-fedora-current.log.html', GINKGO_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(GINKGO_FAILURE, result.stdout)
def test_bats_log(self):
log = self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(BATS_FAILURE, result.stdout)
def test_all_given_logs_are_summarized(self):
"""the workflow passes a glob, none of the matches may be dropped"""
result = self.summarize(
self.write_log('int-local-root-fedora-current.log.html', GINKGO_LOG),
self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG))
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(GINKGO_FAILURE, result.stdout)
self.assertIn(BATS_FAILURE, result.stdout)
self.assertLess(result.stdout.index(GINKGO_FAILURE),
result.stdout.index(BATS_FAILURE),
msg="logs must be reported in the order given")
def test_logs_are_named_only_when_there_is_more_than_one(self):
"""a single log needs no heading, several would run into each other"""
one = self.write_log('int-local-root-fedora-current.log.html', GINKGO_LOG)
self.assertNotIn('###', self.summarize(one).stdout)
two = self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG)
stdout = self.summarize(one, two).stdout
self.assertIn(f"### {os.path.basename(one)}", stdout)
self.assertIn(f"### {os.path.basename(two)}", stdout)
def test_ginkgo_suite_run_without_parallel(self):
"""without -p the stack trace div is the only mark of a failure"""
log = self.write_log('bindings-root-fedora-current.log.html', NO_PARALLEL_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(GINKGO_FAILURE, result.stdout)
def test_logrus_error_does_not_mark_a_passing_test(self):
"""that class also lands on error-level logrus lines, as a span"""
log = self.write_log('bindings-root-fedora-current.log.html', NO_PARALLEL_LOG)
self.assertNotIn(PASSING_TEST_NOISE, self.summarize(log).stdout)
def test_bats_log_holding_a_logrus_error(self):
"""that class must not pull a bats log over to the ginkgo parser"""
log = self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(BATS_FAILURE, result.stdout)
def test_nested_and_unclosed_failure_divs(self):
"""logformatter nests the trace div and leaves some of them open"""
log = self.write_log('bindings-root-fedora-current.log.html',
NESTED_FAILURES_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(FIRST_NESTED, result.stdout)
self.assertIn(SECOND_NESTED, result.stdout)
def test_an_unreadable_log_between_two_good_ones(self):
"""one bad path must not cost us the logs listed after it"""
first = self.write_log('int-local-root-fedora-current.log.html', GINKGO_LOG)
empty = self.write_log('unit-fedora-current.log.html', '')
last = self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG)
result = self.summarize(first, os.path.join(self.tmpdir.name, '*.html'),
empty, last)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(GINKGO_FAILURE, result.stdout)
self.assertIn(BATS_FAILURE, result.stdout)
self.assertLess(result.stdout.index(GINKGO_FAILURE),
result.stdout.index(BATS_FAILURE))
self.assertIn('skipping', result.stderr)
def test_every_failure_block_is_reported(self):
"""ginkgo logs hold one block per failing test, not just one in total"""
second = GINKGO_FAILURE.replace('125', '126')
log = self.write_log('int-local-root-fedora-current.log.html',
GINKGO_LOG + GINKGO_LOG.replace(GINKGO_FAILURE, second))
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(GINKGO_FAILURE, result.stdout)
self.assertIn(second, result.stdout)
def test_log_without_failures_says_nothing(self):
"""an empty code fence is noise, the step summary is better off blank"""
log = self.write_log('build-fedora-current.log.html', UNMARKED_LOG)
result = self.summarize(log)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertEqual(result.stdout, '')
def test_log_holding_invalid_utf8(self):
"""logformatter reads through ':utf8', which lets raw bytes past"""
path = os.path.join(self.tmpdir.name, 'int-local-root-fedora-current.log.html')
with open(path, 'wb') as log:
log.write(GINKGO_LOG.encode().replace(b'exit status 125',
b'exit status \xff\xfe125'))
result = self.summarize(path)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn('Expected error to be nil', result.stdout)
def test_logformatter_still_emits_the_classes_we_look_for(self):
"""the layout is detected by classes logformatter has to keep emitting"""
with open(LOGFORMATTER) as source:
body = source.read()
self.assertIn(GINKGO_CLASS, body)
self.assertIn(GINKGO_CLASS_PREFIX, body)
def test_unreadable_log_is_skipped(self):
"""jobs without any html log hand over an unexpanded glob"""
result = self.summarize(
os.path.join(self.tmpdir.name, '*.html'),
self.write_log('sys-local-root-fedora-current.log.html', BATS_LOG))
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn(BATS_FAILURE, result.stdout)
self.assertIn('skipping', result.stderr)
def test_no_logs_given(self):
result = self.summarize()
self.assertEqual(result.returncode, 2)
self.assertIn('usage', result.stderr)
if __name__ == "__main__":
unittest.main()