| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20200611224028.3275174-7-keescook@chromium.org>
Date: Thu, 11 Jun 2020 15:40:27 -0700
From: Kees Cook <keescook@...omium.org>
To: Shuah Khan <shuah@...nel.org>
Cc: Kees Cook <keescook@...omium.org>,
Andy Lutomirski <luto@...capital.net>,
Will Drewry <wad@...omium.org>,
linux-kselftest@...r.kernel.org,
Christian Brauner <christian@...uner.io>,
Greg Kroah-Hartman <gregkh@...uxfoundation.org>,
linux-kernel@...r.kernel.org
Subject: [PATCH 6/7] selftests/harness: Fully track XFAIL reports
Plumb XFAIL conditions up into TAP reporting and test counts.
Cc: Shuah Khan <shuah@...nel.org>
Cc: Andy Lutomirski <luto@...capital.net>
Cc: Will Drewry <wad@...omium.org>
Cc: linux-kselftest@...r.kernel.org
Signed-off-by: Kees Cook <keescook@...omium.org>
---
tools/testing/selftests/kselftest.h | 17 ++++++-
tools/testing/selftests/kselftest_harness.h | 54 ++++++++++++++++-----
2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h
index 3f0d236ca2e4..9b4efdbb07f6 100644
--- a/tools/testing/selftests/kselftest.h
+++ b/tools/testing/selftests/kselftest.h
@@ -17,12 +17,13 @@
*
* ksft_print_msg(fmt, ...);
*
- * and finally report the pass/fail/skip state of the test with one of:
+ * and finally report the pass/fail/skip/xfail state of the test with one of:
*
* ksft_test_result(condition, fmt, ...);
* ksft_test_result_pass(fmt, ...);
* ksft_test_result_fail(fmt, ...);
* ksft_test_result_skip(fmt, ...);
+ * ksft_test_result_xfail(fmt, ...);
* ksft_test_result_error(fmt, ...);
*
* When all tests are finished, clean up and exit the program with one of:
@@ -163,7 +164,19 @@ static inline void ksft_test_result_fail(const char *msg, ...)
ksft_test_result_fail(fmt, ##__VA_ARGS__);\
} while (0)
-/* TODO: add ksft_test_result_xfail() */
+static inline void ksft_test_result_xfail(const char *msg, ...)
+{
+ int saved_errno = errno;
+ va_list args;
+
+ ksft_cnt.ksft_xfail++;
+
+ va_start(args, msg);
+ printf("ok %d # XFAIL ", ksft_test_num());
+ errno = saved_errno;
+ vprintf(msg, args);
+ va_end(args);
+}
static inline void ksft_test_result_skip(const char *msg, ...)
{
diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h
index f8f7e47c739a..6b06930468e5 100644
--- a/tools/testing/selftests/kselftest_harness.h
+++ b/tools/testing/selftests/kselftest_harness.h
@@ -126,8 +126,8 @@
fprintf(TH_LOG_STREAM, "# XFAIL " fmt "\n", \
##__VA_ARGS__); \
} \
- /* TODO: find a way to pass xfail to test runner process. */ \
_metadata->passed = 1; \
+ _metadata->xfail = 1; \
_metadata->trigger = 0; \
statement; \
} while (0)
@@ -777,6 +777,7 @@ struct __test_metadata {
struct __fixture_metadata *fixture;
int termsig;
int passed;
+ int xfail; /* did XFAIL get used? */
int trigger; /* extra handler after the evaluation */
int timeout; /* seconds to wait for test timeout */
bool timed_out; /* did this test timeout instead of exiting? */
@@ -866,17 +867,31 @@ void __wait_for_test(struct __test_metadata *t)
fprintf(TH_LOG_STREAM,
"# %s: Test terminated by timeout\n", t->name);
} else if (WIFEXITED(status)) {
- t->passed = t->termsig == -1 ? !WEXITSTATUS(status) : 0;
if (t->termsig != -1) {
+ t->passed = 0;
fprintf(TH_LOG_STREAM,
"# %s: Test exited normally instead of by signal (code: %d)\n",
t->name,
WEXITSTATUS(status));
- } else if (!t->passed) {
- fprintf(TH_LOG_STREAM,
- "# %s: Test failed at step #%d\n",
- t->name,
- WEXITSTATUS(status));
+ } else {
+ switch (WEXITSTATUS(status)) {
+ /* Success */
+ case 0:
+ t->passed = 1;
+ break;
+ /* XFAIL */
+ case 255:
+ t->passed = 1;
+ t->xfail = 1;
+ break;
+ /* Other failure, assume step report. */
+ default:
+ t->passed = 0;
+ fprintf(TH_LOG_STREAM,
+ "# %s: Test failed at step #%d\n",
+ t->name,
+ WEXITSTATUS(status));
+ }
}
} else if (WIFSIGNALED(status)) {
t->passed = 0;
@@ -906,6 +921,7 @@ void __run_test(struct __fixture_metadata *f,
{
/* reset test struct */
t->passed = 1;
+ t->xfail = 0;
t->trigger = 0;
t->step = 0;
t->no_print = 0;
@@ -918,15 +934,31 @@ void __run_test(struct __fixture_metadata *f,
t->passed = 0;
} else if (t->pid == 0) {
t->fn(t, variant);
- /* return the step that failed or 0 */
- _exit(t->passed ? 0 : t->step);
+ /* Make sure step doesn't get lost in reporting */
+ if (t->step >= 255) {
+ ksft_print_msg("Too many test steps (%u)!?\n", t->step);
+ t->step = 254;
+ }
+ /* Use 255 for XFAIL */
+ if (t->xfail)
+ _exit(255);
+ /* Pass is exit 0 */
+ if (t->passed)
+ _exit(0);
+ /* Something else happened, report the step. */
+ _exit(t->step);
} else {
__wait_for_test(t);
}
ksft_print_msg(" %4s %s%s%s.%s\n", t->passed ? "OK" : "FAIL",
f->name, variant->name[0] ? "." : "", variant->name, t->name);
- ksft_test_result(t->passed, "%s%s%s.%s\n",
- f->name, variant->name[0] ? "." : "", variant->name, t->name);
+
+ if (t->xfail)
+ ksft_test_result_xfail("%s%s%s.%s\n",
+ f->name, variant->name[0] ? "." : "", variant->name, t->name);
+ else
+ ksft_test_result(t->passed, "%s%s%s.%s\n",
+ f->name, variant->name[0] ? "." : "", variant->name, t->name);
}
static int test_harness_run(int __attribute__((unused)) argc,
--
2.25.1
Powered by blists - more mailing lists