lists.openwall.net   lists  /  announce  owl-users  owl-dev  john-users  john-dev  passwdqc-users  yescrypt  popa3d-users  /  oss-security  kernel-hardening  musl  sabotage  tlsify  passwords  /  crypt-dev  xvendor  /  Bugtraq  Full-Disclosure  linux-kernel  linux-netdev  linux-ext4  linux-hardening  linux-cve-announce  PHC 
Open Source and information security mailing list archives
 
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <CAM0EoMnyjyLYokQ7kC6Zbicq3CTwiBE0oueQ5bDJRFhkPN8W3g@mail.gmail.com>
Date:   Wed, 26 Oct 2022 01:08:47 -0400
From:   Jamal Hadi Salim <jhs@...atatu.com>
To:     Victor Nogueira <victor@...atatu.com>
Cc:     davem@...emloft.net, jiri@...nulli.us, kuba@...nel.org,
        netdev@...r.kernel.org, xiyou.wangcong@...il.com,
        edumazet@...gle.com, pabeni@...hat.com,
        Jeremy Carter <jeremy@...atatu.com>,
        Jeremy Carter <jeremy@...emycarter.ca>
Subject: Re: [PATCH] selftests: tc-testing: Add matchJSON to tdc

On Mon, Oct 24, 2022 at 7:31 AM Victor Nogueira <victor@...atatu.com> wrote:
>
> This allows the use of a matchJSON field in tests to match
> against JSON output from the command under test, if that
> command outputs JSON.
>
> You specify what you want to match against as a JSON array
> or object in the test's matchJSON field. You can leave out
> any fields you don't want to match against that are present
> in the output and they will be skipped.
>
> An example matchJSON value would look like this:
>
> "matchJSON": [
>   {
>     "Value": {
>       "neighIP": {
>         "family": 4,
>         "addr": "AQIDBA==",
>         "width": 32
>       },
>       "nsflags": 142,
>       "ncflags": 0,
>       "LLADDR": "ESIzRFVm"
>     }
>   }
> ]
>
> The real output from the command under test might have some
> extra fields that we don't care about for matching, and
> since we didn't include them in our matchJSON value, those
> fields will not be attempted to be matched. If everything
> we included above has the same values as the real command
> output, the test will pass.
>
> The matchJSON field's type must be the same as the command
> output's type, otherwise the test will fail. So if the
> command outputs an array, then the value of matchJSON must
> also be an array.
>
> If matchJSON is an array, it must not contain more elements
> than the command output's array, otherwise the test will
> fail.
>
> Signed-off-by: Jeremy Carter <jeremy@...atatu.com>
> Signed-off-by: Victor Nogueira <victor@...atatu.com>

Acked-by: Jamal Hadi Salim <jhs@...atatu.com>

cheers,
jamal

> ---
>  tools/testing/selftests/tc-testing/tdc.py | 125 ++++++++++++++++++++--
>  1 file changed, 118 insertions(+), 7 deletions(-)
>
> diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
> index ee22e3447..7bd94f8e4 100755
> --- a/tools/testing/selftests/tc-testing/tdc.py
> +++ b/tools/testing/selftests/tc-testing/tdc.py
> @@ -246,6 +246,110 @@ def prepare_env(args, pm, stage, prefix, cmdlist, output = None):
>                  stage, output,
>                  '"{}" did not complete successfully'.format(prefix))
>
> +def verify_by_json(procout, res, tidx, args, pm):
> +    try:
> +        outputJSON = json.loads(procout)
> +    except json.JSONDecodeError:
> +        res.set_result(ResultState.fail)
> +        res.set_failmsg('Cannot decode verify command\'s output. Is it JSON?')
> +        return res
> +
> +    matchJSON = json.loads(json.dumps(tidx['matchJSON']))
> +
> +    if type(outputJSON) != type(matchJSON):
> +        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {} '
> +        failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
> +        res.set_result(ResultState.fail)
> +        res.set_failmsg(failmsg)
> +        return res
> +
> +    if len(matchJSON) > len(outputJSON):
> +        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
> +        failmsg = failmsg.format(len(outputJSON), outputJSON, len(matchJSON), matchJSON)
> +        res.set_result(ResultState.fail)
> +        res.set_failmsg(failmsg)
> +        return res
> +    res = find_in_json(res, outputJSON, matchJSON, 0)
> +
> +    return res
> +
> +def find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> +    if res.get_result() == ResultState.fail:
> +        return res
> +
> +    if type(matchJSONVal) == list:
> +        res = find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +
> +    elif type(matchJSONVal) == dict:
> +        res = find_in_json_dict(res, outputJSONVal, matchJSONVal)
> +    else:
> +        res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +
> +    if res.get_result() != ResultState.fail:
> +        res.set_result(ResultState.success)
> +        return res
> +
> +    return res
> +
> +def find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> +    if (type(matchJSONVal) != type(outputJSONVal)):
> +        failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
> +        failmsg = failmsg.format(outputJSONVal, matchJSONVal)
> +        res.set_result(ResultState.fail)
> +        res.set_failmsg(failmsg)
> +        return res
> +
> +    if len(matchJSONVal) > len(outputJSONVal):
> +        failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
> +        failmsg = failmsg.format(len(outputJSONVal), outputJSONVal, len(matchJSONVal), matchJSONVal)
> +        res.set_result(ResultState.fail)
> +        res.set_failmsg(failmsg)
> +        return res
> +
> +    for matchJSONIdx, matchJSONVal in enumerate(matchJSONVal):
> +        res = find_in_json(res, outputJSONVal[matchJSONIdx], matchJSONVal,
> +                           matchJSONKey)
> +    return res
> +
> +def find_in_json_dict(res, outputJSONVal, matchJSONVal):
> +    for matchJSONKey, matchJSONVal in matchJSONVal.items():
> +        if type(outputJSONVal) == dict:
> +            if matchJSONKey not in outputJSONVal:
> +                failmsg = 'Key not found in json output: {}: {}\nMatching against output: {}'
> +                failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal)
> +                res.set_result(ResultState.fail)
> +                res.set_failmsg(failmsg)
> +                return res
> +
> +        else:
> +            failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
> +            failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
> +            res.set_result(ResultState.fail)
> +            res.set_failmsg(failmsg)
> +            return rest
> +
> +        if type(outputJSONVal) == dict and (type(outputJSONVal[matchJSONKey]) == dict or
> +                type(outputJSONVal[matchJSONKey]) == list):
> +            if len(matchJSONVal) > 0:
> +                res = find_in_json(res, outputJSONVal[matchJSONKey], matchJSONVal, matchJSONKey)
> +            # handling corner case where matchJSONVal == [] or matchJSONVal == {}
> +            else:
> +                res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +        else:
> +            res = find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +    return res
> +
> +def find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> +    if matchJSONKey in outputJSONVal:
> +        if matchJSONVal != outputJSONVal[matchJSONKey]:
> +            failmsg = 'Value doesn\'t match: {}: {} != {}\nMatching against output: {}'
> +            failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal[matchJSONKey], outputJSONVal)
> +            res.set_result(ResultState.fail)
> +            res.set_failmsg(failmsg)
> +            return res
> +
> +    return res
> +
>  def run_one_test(pm, args, index, tidx):
>      global NAMES
>      result = True
> @@ -292,16 +396,22 @@ def run_one_test(pm, args, index, tidx):
>      else:
>          if args.verbose > 0:
>              print('-----> verify stage')
> -        match_pattern = re.compile(
> -            str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
>          (p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
>          if procout:
> -            match_index = re.findall(match_pattern, procout)
> -            if len(match_index) != int(tidx["matchCount"]):
> -                res.set_result(ResultState.fail)
> -                res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
> +            if 'matchJSON' in tidx:
> +                verify_by_json(procout, res, tidx, args, pm)
> +            elif 'matchPattern' in tidx:
> +                match_pattern = re.compile(
> +                    str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
> +                match_index = re.findall(match_pattern, procout)
> +                if len(match_index) != int(tidx["matchCount"]):
> +                    res.set_result(ResultState.fail)
> +                    res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
> +                else:
> +                    res.set_result(ResultState.success)
>              else:
> -                res.set_result(ResultState.success)
> +                res.set_result(ResultState.fail)
> +                res.set_failmsg('Must specify a match option: matchJSON or matchPattern\n{}'.format(procout))
>          elif int(tidx["matchCount"]) != 0:
>              res.set_result(ResultState.fail)
>              res.set_failmsg('No output generated by verify command.')
> @@ -365,6 +475,7 @@ def test_runner(pm, args, filtered_tests):
>              res.set_result(ResultState.skip)
>              res.set_errormsg(errmsg)
>              tsr.add_resultdata(res)
> +            index += 1
>              continue
>          try:
>              badtest = tidx  # in case it goes bad
> --
> 2.25.1
>

Powered by blists - more mailing lists