netdev - Re: [PATCH net-next 2/7] tools: tc-testing: Refactor test-runner

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:   Wed, 14 Feb 2018 17:44:21 -0500
From:   Lucas Bates <lucasb@...atatu.com>
To:     "Brenda J. Butler" <bjb@...atatu.com>
Cc:     davem@...emloft.net, kernel@...atatu.com,
        Cong Wang <xiyou.wangcong@...il.com>,
        Jiri Pirko <jiri@...nulli.us>, Chris Mi <chrism@...lanox.com>,
        Linux Kernel Network Developers <netdev@...r.kernel.org>
Subject: Re: [PATCH net-next 2/7] tools: tc-testing: Refactor test-runner

On Wed, Feb 14, 2018 at 2:09 PM, Brenda J. Butler <bjb@...atatu.com> wrote:
> Split the test_runner function into the loop part (test_runner)
> and the contents (run_one_test) for maintainability.
> It makes it a little easier to catch exceptions
> in an individual test, and keep going (and flush a bunch
> of tap results for the skipped tests).
>
> Signed-off-by: Brenda J. Butler <bjb@...atatu.com>

Acked-by: Lucas Bates <lucasb@...atatu.com>

> ---
>  tools/testing/selftests/tc-testing/tdc.py | 81 ++++++++++++++++++++-----------
>  1 file changed, 52 insertions(+), 29 deletions(-)
>
> diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
> index ef3a8881e458..a2624eda34db 100755
> --- a/tools/testing/selftests/tc-testing/tdc.py
> +++ b/tools/testing/selftests/tc-testing/tdc.py
> @@ -85,9 +85,42 @@ def prepare_env(cmdlist):
>              print("\nError message:")
>              print(foutput)
>              print("\nAborting test run.")
> -            ns_destroy()
> -            exit(1)
> +            # ns_destroy()
> +            raise Exception('prepare_env did not complete successfully')
> +
> +def run_one_test(index, tidx):
> +    result = True
> +    tresult = ""
> +    tap = ""
> +    print("Test " + tidx["id"] + ": " + tidx["name"])
> +    prepare_env(tidx["setup"])
> +    (p, procout) = exec_cmd(tidx["cmdUnderTest"])
> +    exit_code = p.returncode
> +
> +    if (exit_code != int(tidx["expExitCode"])):
> +        result = False
> +        print("exit:", exit_code, int(tidx["expExitCode"]))
> +        print(procout)
> +    else:
> +        match_pattern = re.compile(str(tidx["matchPattern"]),
> +                                   re.DOTALL | re.MULTILINE)
> +        (p, procout) = exec_cmd(tidx["verifyCmd"])
> +        match_index = re.findall(match_pattern, procout)
> +        if len(match_index) != int(tidx["matchCount"]):
> +            result = False
> +
> +    if not result:
> +        tresult += "not "
> +    tresult += "ok {} - {} # {}\n".format(str(index), tidx['id'], tidx["name"])
> +    tap += tresult
>
> +    if result == False:
> +        tap += procout
> +
> +    prepare_env(tidx["teardown"])
> +    index += 1
> +
> +    return tap
>
>  def test_runner(filtered_tests, args):
>      """
> @@ -104,37 +137,27 @@ def test_runner(filtered_tests, args):
>      tap = str(index) + ".." + str(tcount) + "\n"
>
>      for tidx in testlist:
> -        result = True
> -        tresult = ""
>          if "flower" in tidx["category"] and args.device == None:
>              continue
> -        print("Test " + tidx["id"] + ": " + tidx["name"])
> -        prepare_env(tidx["setup"])
> -        (p, procout) = exec_cmd(tidx["cmdUnderTest"])
> -        exit_code = p.returncode
> -
> -        if (exit_code != int(tidx["expExitCode"])):
> -            result = False
> -            print("exit:", exit_code, int(tidx["expExitCode"]))
> -            print(procout)
> -        else:
> -            match_pattern = re.compile(str(tidx["matchPattern"]), re.DOTALL)
> -            (p, procout) = exec_cmd(tidx["verifyCmd"])
> -            match_index = re.findall(match_pattern, procout)
> -            if len(match_index) != int(tidx["matchCount"]):
> -                result = False
> -
> -        if result == True:
> -            tresult += "ok "
> -        else:
> -            tresult += "not ok "
> -        tap += tresult + str(index) + " " + tidx["id"] + " " + tidx["name"] + "\n"
> +        try:
> +            badtest = tidx  # in case it goes bad
> +            tap += run_one_test(index, tidx)
> +        except Exception as ee:
> +            print('Exception {} (caught in test_runner, running test {} {} {})'.
> +                  format(ee, index, tidx['id'], tidx['name']))
> +            break
> +        index += 1
>
> -        if result == False:
> -            tap += procout
> +    count = index
> +    tap += 'about to flush the tap output if tests need to be skipped\n'
> +    if tcount + 1 != index:
> +        for tidx in testlist[index - 1:]:
> +            msg = 'skipped - previous setup or teardown failed'
> +            tap += 'ok {} - {} # {} {} {} \n'.format(
> +                count, tidx['id'], msg, index, badtest.get('id', '--Unknown--'))
> +            count += 1
>
> -        prepare_env(tidx["teardown"])
> -        index += 1
> +    tap += 'done flushing skipped test tap output\n'
>
>      return tap
>
> --
> 2.15.1
>