netdev - [PATCH net-next 2/7] tools: tc-testing: Refactor test-runner

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20180214190925.20195-3-bjb@mojatatu.com>
Date:   Wed, 14 Feb 2018 14:09:20 -0500
From:   "Brenda J. Butler" <bjb@...atatu.com>
To:     davem@...emloft.net
Cc:     kernel@...atatu.com, xiyou.wangcong@...il.com, jiri@...nulli.us,
        chrism@...lanox.com, lucasb@...atatu.com, netdev@...r.kernel.org,
        "Brenda J. Butler" <bjb@...atatu.com>
Subject: [PATCH net-next 2/7] tools: tc-testing: Refactor test-runner

Split the test_runner function into the loop part (test_runner)
and the contents (run_one_test) for maintainability.
It makes it a little easier to catch exceptions
in an individual test, and keep going (and flush a bunch
of tap results for the skipped tests).

Signed-off-by: Brenda J. Butler <bjb@...atatu.com>
---
 tools/testing/selftests/tc-testing/tdc.py | 81 ++++++++++++++++++++-----------
 1 file changed, 52 insertions(+), 29 deletions(-)

diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index ef3a8881e458..a2624eda34db 100755
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -85,9 +85,42 @@ def prepare_env(cmdlist):
             print("\nError message:")
             print(foutput)
             print("\nAborting test run.")
-            ns_destroy()
-            exit(1)
+            # ns_destroy()
+            raise Exception('prepare_env did not complete successfully')
+
+def run_one_test(index, tidx):
+    result = True
+    tresult = ""
+    tap = ""
+    print("Test " + tidx["id"] + ": " + tidx["name"])
+    prepare_env(tidx["setup"])
+    (p, procout) = exec_cmd(tidx["cmdUnderTest"])
+    exit_code = p.returncode
+
+    if (exit_code != int(tidx["expExitCode"])):
+        result = False
+        print("exit:", exit_code, int(tidx["expExitCode"]))
+        print(procout)
+    else:
+        match_pattern = re.compile(str(tidx["matchPattern"]),
+                                   re.DOTALL | re.MULTILINE)
+        (p, procout) = exec_cmd(tidx["verifyCmd"])
+        match_index = re.findall(match_pattern, procout)
+        if len(match_index) != int(tidx["matchCount"]):
+            result = False
+
+    if not result:
+        tresult += "not "
+    tresult += "ok {} - {} # {}\n".format(str(index), tidx['id'], tidx["name"])
+    tap += tresult
 
+    if result == False:
+        tap += procout
+
+    prepare_env(tidx["teardown"])
+    index += 1
+
+    return tap
 
 def test_runner(filtered_tests, args):
     """
@@ -104,37 +137,27 @@ def test_runner(filtered_tests, args):
     tap = str(index) + ".." + str(tcount) + "\n"
 
     for tidx in testlist:
-        result = True
-        tresult = ""
         if "flower" in tidx["category"] and args.device == None:
             continue
-        print("Test " + tidx["id"] + ": " + tidx["name"])
-        prepare_env(tidx["setup"])
-        (p, procout) = exec_cmd(tidx["cmdUnderTest"])
-        exit_code = p.returncode
-
-        if (exit_code != int(tidx["expExitCode"])):
-            result = False
-            print("exit:", exit_code, int(tidx["expExitCode"]))
-            print(procout)
-        else:
-            match_pattern = re.compile(str(tidx["matchPattern"]), re.DOTALL)
-            (p, procout) = exec_cmd(tidx["verifyCmd"])
-            match_index = re.findall(match_pattern, procout)
-            if len(match_index) != int(tidx["matchCount"]):
-                result = False
-
-        if result == True:
-            tresult += "ok "
-        else:
-            tresult += "not ok "
-        tap += tresult + str(index) + " " + tidx["id"] + " " + tidx["name"] + "\n"
+        try:
+            badtest = tidx  # in case it goes bad
+            tap += run_one_test(index, tidx)
+        except Exception as ee:
+            print('Exception {} (caught in test_runner, running test {} {} {})'.
+                  format(ee, index, tidx['id'], tidx['name']))
+            break
+        index += 1
 
-        if result == False:
-            tap += procout
+    count = index
+    tap += 'about to flush the tap output if tests need to be skipped\n'
+    if tcount + 1 != index:
+        for tidx in testlist[index - 1:]:
+            msg = 'skipped - previous setup or teardown failed'
+            tap += 'ok {} - {} # {} {} {} \n'.format(
+                count, tidx['id'], msg, index, badtest.get('id', '--Unknown--'))
+            count += 1
 
-        prepare_env(tidx["teardown"])
-        index += 1
+    tap += 'done flushing skipped test tap output\n'
 
     return tap
 
-- 
2.15.1