| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20221103174740.3492603-2-dlatypov@google.com>
Date: Thu, 3 Nov 2022 10:47:39 -0700
From: Daniel Latypov <dlatypov@...gle.com>
To: brendanhiggins@...gle.com, davidgow@...gle.com
Cc: rmoar@...gle.com, linux-kernel@...r.kernel.org,
kunit-dev@...glegroups.com, linux-kselftest@...r.kernel.org,
skhan@...uxfoundation.org, Daniel Latypov <dlatypov@...gle.com>
Subject: [PATCH v2 2/3] kunit: tool: unit tests all check parser errors,
standardize formatting a bit
Let's verify that the parser isn't reporting any errors for valid
inputs.
This change also
* does result.status checking on one line
* makes sure we consistently do it outside of the `with` block
Signed-off-by: Daniel Latypov <dlatypov@...gle.com>
Reviewed-by: David Gow <davidgow@...gle.com>
---
v1 -> v2: just rebased onto linux-kselftest/kunit
---
tools/testing/kunit/kunit_tool_test.py | 93 +++++++++++---------------
1 file changed, 38 insertions(+), 55 deletions(-)
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 440a273f1d21..5e3429a1202b 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -136,33 +136,29 @@ class KUnitParserTest(unittest.TestCase):
all_passed_log = test_data_path('test_is_test_passed-all_passed.log')
with open(all_passed_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual(result.counts.errors, 0)
def test_parse_successful_nested_tests_log(self):
all_passed_log = test_data_path('test_is_test_passed-all_passed_nested.log')
with open(all_passed_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual(result.counts.errors, 0)
def test_kselftest_nested(self):
kselftest_log = test_data_path('test_is_test_passed-kselftest.log')
with open(kselftest_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual(result.counts.errors, 0)
def test_parse_failed_test_log(self):
failed_log = test_data_path('test_is_test_passed-failure.log')
with open(failed_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.FAILURE,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status)
+ self.assertEqual(result.counts.errors, 0)
def test_no_header(self):
empty_log = test_data_path('test_is_test_passed-no_tests_run_no_header.log')
@@ -170,9 +166,8 @@ class KUnitParserTest(unittest.TestCase):
result = kunit_parser.parse_run_tests(
kunit_parser.extract_tap_lines(file.readlines()))
self.assertEqual(0, len(result.subtests))
- self.assertEqual(
- kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS, result.status)
+ self.assertEqual(result.counts.errors, 1)
def test_missing_test_plan(self):
missing_plan_log = test_data_path('test_is_test_passed-'
@@ -183,9 +178,7 @@ class KUnitParserTest(unittest.TestCase):
file.readlines()))
# A missing test plan is not an error.
self.assertEqual(result.counts, kunit_parser.TestCounts(passed=10, errors=0))
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
def test_no_tests(self):
header_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log')
@@ -193,9 +186,8 @@ class KUnitParserTest(unittest.TestCase):
result = kunit_parser.parse_run_tests(
kunit_parser.extract_tap_lines(file.readlines()))
self.assertEqual(0, len(result.subtests))
- self.assertEqual(
- kunit_parser.TestStatus.NO_TESTS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.NO_TESTS, result.status)
+ self.assertEqual(result.counts.errors, 1)
def test_no_tests_no_plan(self):
no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log')
@@ -206,7 +198,7 @@ class KUnitParserTest(unittest.TestCase):
self.assertEqual(
kunit_parser.TestStatus.NO_TESTS,
result.subtests[0].subtests[0].status)
- self.assertEqual(1, result.counts.errors)
+ self.assertEqual(result.counts, kunit_parser.TestCounts(passed=1, errors=1))
def test_no_kunit_output(self):
@@ -218,6 +210,7 @@ class KUnitParserTest(unittest.TestCase):
print_mock.assert_any_call(StrContains('could not find any KTAP output!'))
print_mock.stop()
self.assertEqual(0, len(result.subtests))
+ self.assertEqual(result.counts.errors, 1)
def test_skipped_test(self):
skipped_log = test_data_path('test_skip_tests.log')
@@ -225,18 +218,16 @@ class KUnitParserTest(unittest.TestCase):
result = kunit_parser.parse_run_tests(file.readlines())
# A skipped test does not fail the whole suite.
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual(result.counts, kunit_parser.TestCounts(passed=4, skipped=1))
def test_skipped_all_tests(self):
skipped_log = test_data_path('test_skip_all_tests.log')
with open(skipped_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SKIPPED,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.status)
+ self.assertEqual(result.counts, kunit_parser.TestCounts(skipped=5))
def test_ignores_hyphen(self):
hyphen_log = test_data_path('test_strip_hyphen.log')
@@ -244,9 +235,7 @@ class KUnitParserTest(unittest.TestCase):
result = kunit_parser.parse_run_tests(file.readlines())
# A skipped test does not fail the whole suite.
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
self.assertEqual(
"sysctl_test",
result.subtests[0].name)
@@ -260,55 +249,49 @@ class KUnitParserTest(unittest.TestCase):
prefix_log = test_data_path('test_config_printk_time.log')
with open(prefix_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(result.counts.errors, 0)
def test_ignores_multiple_prefixes(self):
prefix_log = test_data_path('test_multiple_prefixes.log')
with open(prefix_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(result.counts.errors, 0)
def test_prefix_mixed_kernel_output(self):
mixed_prefix_log = test_data_path('test_interrupted_tap_output.log')
with open(mixed_prefix_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(result.counts.errors, 0)
def test_prefix_poundsign(self):
pound_log = test_data_path('test_pound_sign.log')
with open(pound_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(result.counts.errors, 0)
def test_kernel_panic_end(self):
panic_log = test_data_path('test_kernel_panic_interrupt.log')
with open(panic_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.TEST_CRASHED,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.TEST_CRASHED, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertGreaterEqual(result.counts.errors, 1)
def test_pound_no_prefix(self):
pound_log = test_data_path('test_pound_no_prefix.log')
with open(pound_log) as file:
result = kunit_parser.parse_run_tests(file.readlines())
- self.assertEqual(
- kunit_parser.TestStatus.SUCCESS,
- result.status)
- self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
+ self.assertEqual(result.counts.errors, 0)
def test_summarize_failures(self):
output = """
--
2.38.1.431.g37b22c650d-goog
Powered by blists - more mailing lists