| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CAD4GDZzWMoxnatNXYbKOphzVZ4NyedD5FtjxF7cgB1ad-wDFWg@mail.gmail.com>
Date: Tue, 17 Jun 2025 18:23:22 +0100
From: Donald Hunter <donald.hunter@...il.com>
To: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
Cc: Linux Doc Mailing List <linux-doc@...r.kernel.org>, Jonathan Corbet <corbet@....net>,
Akira Yokosawa <akiyks@...il.com>, Breno Leitao <leitao@...ian.org>,
"David S. Miller" <davem@...emloft.net>, Eric Dumazet <edumazet@...gle.com>,
Ignacio Encinas Rubio <ignacio@...cinas.com>, Jan Stancek <jstancek@...hat.com>,
Marco Elver <elver@...gle.com>, Paolo Abeni <pabeni@...hat.com>, Ruben Wauters <rubenru09@....com>,
Shuah Khan <skhan@...uxfoundation.org>, joel@...lfernandes.org,
linux-kernel-mentees@...ts.linux.dev, linux-kernel@...r.kernel.org,
lkmm@...ts.linux.dev, netdev@...r.kernel.org, peterz@...radead.org,
stern@...land.harvard.edu
Subject: Re: [PATCH v5 10/15] docs: sphinx: add a parser for yaml files for
Netlink specs
On Tue, 17 Jun 2025 at 17:00, Mauro Carvalho Chehab
<mchehab+huawei@...nel.org> wrote:
> >
> > (2) is cleaner and faster, but (1) is easier to implement on an
> > already-existing code.
>
> The logic below implements (1). This seems to be the easiest way for
> pyyaml. I will submit as 2 separate patches at the end of the next
> version.
>
> Please notice that I didn't check yet for the "quality" of the
> line numbers. Some tweaks could be needed later on.
Thanks for working on this. I suppose we might be able to work on an
evolution from (1) to (2) in a followup piece of work?
> Regards,
> Mauro
>
> ---
>
> From 750daebebadcd156b5fe9b516f4fae4bd42b9d2c Mon Sep 17 00:00:00 2001
> From: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
> Date: Tue, 17 Jun 2025 17:54:03 +0200
> Subject: [PATCH] docs: parser_yaml.py: add support for line numbers from the
> parser
>
> Instead of printing line numbers from the temp converted ReST
> file, get them from the original source.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
>
> diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
> index 635945e1c5ba..15c642fc0bd5 100755
> --- a/Documentation/sphinx/parser_yaml.py
> +++ b/Documentation/sphinx/parser_yaml.py
> @@ -29,6 +29,8 @@ class YamlParser(Parser):
>
> netlink_parser = YnlDocGenerator()
>
> + re_lineno = re.compile(r"\.\. LINENO ([0-9]+)$")
> +
> def do_parse(self, inputstring, document, msg):
> """Parse YAML and generate a document tree."""
>
> @@ -38,8 +40,14 @@ class YamlParser(Parser):
>
> try:
> # Parse message with RSTParser
> - for i, line in enumerate(msg.split('\n')):
> - result.append(line, document.current_source, i)
> + lineoffset = 0;
> + for line in msg.split('\n'):
> + match = self.re_lineno.match(line)
> + if match:
> + lineoffset = int(match.group(1))
> + continue
> +
> + result.append(line, document.current_source, lineoffset)
>
> rst_parser = RSTParser()
> rst_parser.parse('\n'.join(result), document)
>
> From 15c1f9db30f3abdce110e19788d87f9fe1417781 Mon Sep 17 00:00:00 2001
> From: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
> Date: Tue, 17 Jun 2025 17:28:04 +0200
> Subject: [PATCH] tools: netlink_yml_parser.py: add line numbers to parsed data
>
> When something goes wrong, we want Sphinx error to point to the
> right line number from the original source, not from the
> processed ReST data.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
>
> diff --git a/tools/net/ynl/pyynl/netlink_yml_parser.py b/tools/net/ynl/pyynl/netlink_yml_parser.py
> index 866551726723..a9d8ab6f2639 100755
> --- a/tools/net/ynl/pyynl/netlink_yml_parser.py
> +++ b/tools/net/ynl/pyynl/netlink_yml_parser.py
> @@ -20,6 +20,16 @@
> from typing import Any, Dict, List
> import yaml
>
> +LINE_STR = '__lineno__'
> +
> +class NumberedSafeLoader(yaml.SafeLoader):
> + """Override the SafeLoader class to add line number to parsed data"""
> +
> + def construct_mapping(self, node):
> + mapping = super().construct_mapping(node)
> + mapping[LINE_STR] = node.start_mark.line
> +
> + return mapping
>
> class RstFormatters:
> """RST Formatters"""
> @@ -127,6 +137,11 @@ class RstFormatters:
> """Return a formatted label"""
> return f".. _{title}:\n\n"
>
> + @staticmethod
> + def rst_lineno(lineno: int) -> str:
> + """Return a lineno comment"""
> + return f".. LINENO {lineno}\n"
> +
> class YnlDocGenerator:
> """YAML Netlink specs Parser"""
>
> @@ -144,6 +159,9 @@ class YnlDocGenerator:
> """Parse 'do' section and return a formatted string"""
> lines = []
> for key in do_dict.keys():
> + if key == LINE_STR:
> + lines.append(self.fmt.rst_lineno(do_dict[key]))
> + continue
> lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1))
> if key in ['request', 'reply']:
> lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n")
> @@ -174,6 +192,10 @@ class YnlDocGenerator:
> lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n")
>
> for key in operation.keys():
> + if key == LINE_STR:
> + lines.append(self.fmt.rst_lineno(operation[key]))
> + continue
> +
> if key in preprocessed:
> # Skip the special fields
> continue
> @@ -233,6 +255,9 @@ class YnlDocGenerator:
> for definition in defs:
> lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"]))
> for k in definition.keys():
> + if k == LINE_STR:
> + lines.append(self.fmt.rst_lineno(definition[k]))
> + continue
> if k in preprocessed + ignored:
> continue
> lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0))
> @@ -268,6 +293,9 @@ class YnlDocGenerator:
> lines.append(self.fmt.rst_subsubsection(attr_line))
>
> for k in attr.keys():
> + if k == LINE_STR:
> + lines.append(self.fmt.rst_lineno(attr[k]))
> + continue
> if k in preprocessed + ignored:
> continue
> if k in linkable:
> @@ -306,6 +334,8 @@ class YnlDocGenerator:
> lines = []
>
> # Main header
> + lineno = obj.get('__lineno__', 0)
> + lines.append(self.fmt.rst_lineno(lineno))
>
> family = obj['name']
>
> @@ -354,7 +384,7 @@ class YnlDocGenerator:
> def parse_yaml_file(self, filename: str) -> str:
> """Transform the YAML specified by filename into an RST-formatted string"""
> with open(filename, "r", encoding="utf-8") as spec_file:
> - yaml_data = yaml.safe_load(spec_file)
> - content = self.parse_yaml(yaml_data)
> + numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader)
> + content = self.parse_yaml(numbered_yaml)
>
> return content
>
Powered by blists - more mailing lists