| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20180223171638.21630-3-bjb@mojatatu.com>
Date: Fri, 23 Feb 2018 12:16:38 -0500
From: "Brenda J. Butler" <bjb@...atatu.com>
To: davem@...emloft.net
Cc: jhs@...atatu.com, xiyou.wangcong@...il.com, jiri@...nulli.us,
chrism@...lanox.com, lucasb@...atatu.com, aring@...atatu.com,
mrv@...atatu.com, andrew@...n.ch, netdev@...r.kernel.org,
"Brenda J. Butler" <bjb@...atatu.com>
Subject: [PATCH v2 net-next 2/2] tools: tc-testing: better error reporting
Do a better job with error handling - in pre- and post-suite,
in pre- and post-case. Show a traceback for errors.
Signed-off-by: Brenda J. Butler <bjb@...atatu.com>
---
tools/testing/selftests/tc-testing/tdc.py | 84 +++++++++++++++++++++++++------
1 file changed, 70 insertions(+), 14 deletions(-)
diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index ab28373dccd7..6c220bc26d9f 100755
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -15,6 +15,7 @@ import importlib
import json
import subprocess
import time
+import traceback
from collections import OrderedDict
from string import Template
@@ -23,6 +24,13 @@ from tdc_helper import *
import TdcPlugin
+
+class PluginMgrTestFail(Exception):
+ def __init__(self, stage, output, message):
+ self.stage = stage
+ self.output = output
+ self.message = message
+
class PluginMgr:
def __init__(self, argparser):
super().__init__()
@@ -135,7 +143,7 @@ def exec_cmd(args, pm, stage, command):
return proc, foutput
-def prepare_env(args, pm, stage, prefix, cmdlist):
+def prepare_env(args, pm, stage, prefix, cmdlist, output = None):
"""
Execute the setup/teardown commands for a test case.
Optionally terminate test execution if the command fails.
@@ -164,7 +172,9 @@ def prepare_env(args, pm, stage, prefix, cmdlist):
print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr)
print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr)
print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr)
- raise Exception('"{}" did not complete successfully'.format(prefix))
+ raise PluginMgrTestFail(
+ stage, output,
+ '"{}" did not complete successfully'.format(prefix))
def run_one_test(pm, args, index, tidx):
result = True
@@ -194,8 +204,11 @@ def run_one_test(pm, args, index, tidx):
match_pattern = re.compile(
str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
(p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
- match_index = re.findall(match_pattern, procout)
- if len(match_index) != int(tidx["matchCount"]):
+ if procout:
+ match_index = re.findall(match_pattern, procout)
+ if len(match_index) != int(tidx["matchCount"]):
+ result = False
+ elif int(tidx["matchCount"]) != 0:
result = False
if not result:
@@ -204,9 +217,12 @@ def run_one_test(pm, args, index, tidx):
tap += tresult
if result == False:
- tap += procout
+ if procout:
+ tap += procout
+ else:
+ tap += 'No output!\n'
- prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'])
+ prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
pm.call_post_case()
index += 1
@@ -227,30 +243,70 @@ def test_runner(pm, args, filtered_tests):
index = 1
tap = str(index) + ".." + str(tcount) + "\n"
badtest = None
+ stage = None
+ emergency_exit = False
+ emergency_exit_message = ''
- pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
-
+ try:
+ pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
+ except Exception as ee:
+ ex_type, ex, ex_tb = sys.exc_info()
+ print('Exception {} {} (caught in pre_suite).'.
+ format(ex_type, ex))
+ # when the extra print statements are uncommented,
+ # the traceback does not appear between them
+ # (it appears way earlier in the tdc.py output)
+ # so don't bother ...
+ # print('--------------------(')
+ # print('traceback')
+ traceback.print_tb(ex_tb)
+ # print('--------------------)')
+ emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
+ emergency_exit = True
+ stage = 'pre-SUITE'
+
+ if emergency_exit:
+ pm.call_post_suite(index)
+ return emergency_exit_message
if args.verbose > 1:
- print('Run tests here')
+ print('give test rig 2 seconds to stabilize')
+ time.sleep(2)
for tidx in testlist:
if "flower" in tidx["category"] and args.device == None:
+ if args.verbose > 1:
+ print('Not executing test {} {} because DEV2 not defined'.
+ format(tidx['id'], tidx['name']))
continue
try:
badtest = tidx # in case it goes bad
tap += run_one_test(pm, args, index, tidx)
- except Exception as ee:
- print('Exception {} (caught in test_runner, running test {} {} {})'.
- format(ee, index, tidx['id'], tidx['name']))
+ except PluginMgrTestFail as pmtf:
+ ex_type, ex, ex_tb = sys.exc_info()
+ stage = pmtf.stage
+ message = pmtf.message
+ output = pmtf.output
+ print(message)
+ print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
+ format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
+ print('---------------')
+ print('traceback')
+ traceback.print_tb(ex_tb)
+ print('---------------')
+ if stage == 'teardown':
+ print('accumulated output for this test:')
+ if pmtf.output:
+ print(pmtf.output)
+ print('---------------')
break
index += 1
# if we failed in setup or teardown,
- # fill in the remaining tests with not ok
+ # fill in the remaining tests with ok-skipped
count = index
tap += 'about to flush the tap output if tests need to be skipped\n'
if tcount + 1 != index:
for tidx in testlist[index - 1:]:
- msg = 'skipped - previous setup or teardown failed'
+ msg = 'skipped - previous {} failed'.format(stage)
tap += 'ok {} - {} # {} {} {}\n'.format(
count, tidx['id'], msg, index, badtest.get('id', '--Unknown--'))
count += 1
--
2.15.1
Powered by blists - more mailing lists