| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CAM0EoMnyjyLYokQ7kC6Zbicq3CTwiBE0oueQ5bDJRFhkPN8W3g@mail.gmail.com>
Date: Wed, 26 Oct 2022 01:08:47 -0400
From: Jamal Hadi Salim <jhs@...atatu.com>
To: Victor Nogueira <victor@...atatu.com>
Cc: davem@...emloft.net, jiri@...nulli.us, kuba@...nel.org,
netdev@...r.kernel.org, xiyou.wangcong@...il.com,
edumazet@...gle.com, pabeni@...hat.com,
Jeremy Carter <jeremy@...atatu.com>,
Jeremy Carter <jeremy@...emycarter.ca>
Subject: Re: [PATCH] selftests: tc-testing: Add matchJSON to tdc
On Mon, Oct 24, 2022 at 7:31 AM Victor Nogueira <victor@...atatu.com> wrote:
>
> This allows the use of a matchJSON field in tests to match
> against JSON output from the command under test, if that
> command outputs JSON.
>
> You specify what you want to match against as a JSON array
> or object in the test's matchJSON field. You can leave out
> any fields you don't want to match against that are present
> in the output and they will be skipped.
>
> An example matchJSON value would look like this:
>
> "matchJSON": [
> {
> "Value": {
> "neighIP": {
> "family": 4,
> "addr": "AQIDBA==",
> "width": 32
> },
> "nsflags": 142,
> "ncflags": 0,
> "LLADDR": "ESIzRFVm"
> }
> }
> ]
>
> The real output from the command under test might have some
> extra fields that we don't care about for matching, and
> since we didn't include them in our matchJSON value, those
> fields will not be attempted to be matched. If everything
> we included above has the same values as the real command
> output, the test will pass.
>
> The matchJSON field's type must be the same as the command
> output's type, otherwise the test will fail. So if the
> command outputs an array, then the value of matchJSON must
> also be an array.
>
> If matchJSON is an array, it must not contain more elements
> than the command output's array, otherwise the test will
> fail.
>
> Signed-off-by: Jeremy Carter <jeremy@...atatu.com>
> Signed-off-by: Victor Nogueira <victor@...atatu.com>
Acked-by: Jamal Hadi Salim <jhs@...atatu.com>
cheers,
jamal
> ---
> tools/testing/selftests/tc-testing/tdc.py | 125 ++++++++++++++++++++--
> 1 file changed, 118 insertions(+), 7 deletions(-)
>
> diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
> index ee22e3447..7bd94f8e4 100755
> --- a/tools/testing/selftests/tc-testing/tdc.py
> +++ b/tools/testing/selftests/tc-testing/tdc.py
> @@ -246,6 +246,110 @@ def prepare_env(args, pm, stage, prefix, cmdlist, output = None):
> stage, output,
> '"{}" did not complete successfully'.format(prefix))
>
> +def verify_by_json(procout, res, tidx, args, pm):
> + try:
> + outputJSON = json.loads(procout)
> + except json.JSONDecodeError:
> + res.set_result(ResultState.fail)
> + res.set_failmsg('Cannot decode verify command\'s output. Is it JSON?')
> + return res
> +
> + matchJSON = json.loads(json.dumps(tidx['matchJSON']))
> +
> + if type(outputJSON) != type(matchJSON):
> + failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {} '
> + failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> +
> + if len(matchJSON) > len(outputJSON):
> + failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
> + failmsg = failmsg.format(len(outputJSON), outputJSON, len(matchJSON), matchJSON)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> + res = find_in_json(res, outputJSON, matchJSON, 0)
> +
> + return res
> +
> +def find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> + if res.get_result() == ResultState.fail:
> + return res
> +
> + if type(matchJSONVal) == list:
> + res = find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +
> + elif type(matchJSONVal) == dict:
> + res = find_in_json_dict(res, outputJSONVal, matchJSONVal)
> + else:
> + res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
> +
> + if res.get_result() != ResultState.fail:
> + res.set_result(ResultState.success)
> + return res
> +
> + return res
> +
> +def find_in_json_list(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> + if (type(matchJSONVal) != type(outputJSONVal)):
> + failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
> + failmsg = failmsg.format(outputJSONVal, matchJSONVal)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> +
> + if len(matchJSONVal) > len(outputJSONVal):
> + failmsg = "Your matchJSON value is an array, and it contains more elements than the command under test\'s output:\ncommand output (length: {}):\n{}\nmatchJSON value (length: {}):\n{}"
> + failmsg = failmsg.format(len(outputJSONVal), outputJSONVal, len(matchJSONVal), matchJSONVal)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> +
> + for matchJSONIdx, matchJSONVal in enumerate(matchJSONVal):
> + res = find_in_json(res, outputJSONVal[matchJSONIdx], matchJSONVal,
> + matchJSONKey)
> + return res
> +
> +def find_in_json_dict(res, outputJSONVal, matchJSONVal):
> + for matchJSONKey, matchJSONVal in matchJSONVal.items():
> + if type(outputJSONVal) == dict:
> + if matchJSONKey not in outputJSONVal:
> + failmsg = 'Key not found in json output: {}: {}\nMatching against output: {}'
> + failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> +
> + else:
> + failmsg = 'Original output and matchJSON value are not the same type: output: {} != matchJSON: {}'
> + failmsg = failmsg.format(type(outputJSON).__name__, type(matchJSON).__name__)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return rest
> +
> + if type(outputJSONVal) == dict and (type(outputJSONVal[matchJSONKey]) == dict or
> + type(outputJSONVal[matchJSONKey]) == list):
> + if len(matchJSONVal) > 0:
> + res = find_in_json(res, outputJSONVal[matchJSONKey], matchJSONVal, matchJSONKey)
> + # handling corner case where matchJSONVal == [] or matchJSONVal == {}
> + else:
> + res = find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey)
> + else:
> + res = find_in_json(res, outputJSONVal, matchJSONVal, matchJSONKey)
> + return res
> +
> +def find_in_json_other(res, outputJSONVal, matchJSONVal, matchJSONKey=None):
> + if matchJSONKey in outputJSONVal:
> + if matchJSONVal != outputJSONVal[matchJSONKey]:
> + failmsg = 'Value doesn\'t match: {}: {} != {}\nMatching against output: {}'
> + failmsg = failmsg.format(matchJSONKey, matchJSONVal, outputJSONVal[matchJSONKey], outputJSONVal)
> + res.set_result(ResultState.fail)
> + res.set_failmsg(failmsg)
> + return res
> +
> + return res
> +
> def run_one_test(pm, args, index, tidx):
> global NAMES
> result = True
> @@ -292,16 +396,22 @@ def run_one_test(pm, args, index, tidx):
> else:
> if args.verbose > 0:
> print('-----> verify stage')
> - match_pattern = re.compile(
> - str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
> (p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
> if procout:
> - match_index = re.findall(match_pattern, procout)
> - if len(match_index) != int(tidx["matchCount"]):
> - res.set_result(ResultState.fail)
> - res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
> + if 'matchJSON' in tidx:
> + verify_by_json(procout, res, tidx, args, pm)
> + elif 'matchPattern' in tidx:
> + match_pattern = re.compile(
> + str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
> + match_index = re.findall(match_pattern, procout)
> + if len(match_index) != int(tidx["matchCount"]):
> + res.set_result(ResultState.fail)
> + res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
> + else:
> + res.set_result(ResultState.success)
> else:
> - res.set_result(ResultState.success)
> + res.set_result(ResultState.fail)
> + res.set_failmsg('Must specify a match option: matchJSON or matchPattern\n{}'.format(procout))
> elif int(tidx["matchCount"]) != 0:
> res.set_result(ResultState.fail)
> res.set_failmsg('No output generated by verify command.')
> @@ -365,6 +475,7 @@ def test_runner(pm, args, filtered_tests):
> res.set_result(ResultState.skip)
> res.set_errormsg(errmsg)
> tsr.add_resultdata(res)
> + index += 1
> continue
> try:
> badtest = tidx # in case it goes bad
> --
> 2.25.1
>
Powered by blists - more mailing lists