| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250617180001.46931ba9@sal.lan>
Date: Tue, 17 Jun 2025 18:00:01 +0200
From: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
To: Donald Hunter <donald.hunter@...il.com>
Cc: Linux Doc Mailing List <linux-doc@...r.kernel.org>, Jonathan Corbet
<corbet@....net>, "Akira Yokosawa" <akiyks@...il.com>, "Breno Leitao"
<leitao@...ian.org>, "David S. Miller" <davem@...emloft.net>, "Eric
Dumazet" <edumazet@...gle.com>, "Ignacio Encinas Rubio"
<ignacio@...cinas.com>, "Jan Stancek" <jstancek@...hat.com>, "Marco Elver"
<elver@...gle.com>, "Paolo Abeni" <pabeni@...hat.com>, "Ruben Wauters"
<rubenru09@....com>, "Shuah Khan" <skhan@...uxfoundation.org>,
joel@...lfernandes.org, linux-kernel-mentees@...ts.linux.dev,
linux-kernel@...r.kernel.org, lkmm@...ts.linux.dev, netdev@...r.kernel.org,
peterz@...radead.org, stern@...land.harvard.edu
Subject: Re: [PATCH v5 10/15] docs: sphinx: add a parser for yaml files for
Netlink specs
Em Tue, 17 Jun 2025 15:40:49 +0200
Mauro Carvalho Chehab <mchehab+huawei@...nel.org> escreveu:
> > > + # Parse message with RSTParser
> > > + for i, line in enumerate(msg.split('\n')):
> > > + result.append(line, document.current_source, i)
> >
> > This has the effect of associating line numbers from the generated ReST
> > with the source .yaml file, right? So errors will be reported against
> > the wrong place in the file. Is there any way to show the cause of the
> > error in the intermediate ReST?
>
> Yes, but this will require modifying the parser. I prefer merging this
> series without such change, and then having a separate changeset
> addressing it.
>
> There are two ways we can do that:
>
> 1. The parser can add a ReST comment with the line number. This
> is what it is done by kerneldoc.py Sphinx extension:
>
> lineoffset = 0
> line_regex = re.compile(r"^\.\. LINENO ([0-9]+)$")
> for line in lines:
> match = line_regex.search(line)
> if match:
> lineoffset = int(match.group(1)) - 1 # sphinx counts lines from 0
> else:
> doc = str(env.srcdir) + "/" + env.docname + ":" + str(self.lineno)
> result.append(line, doc + ": " + filename, lineoffset)
> lineoffset += 1
>
> I kept the same way after its conversion to Python, as right now,
> it supports both a Python class and a command lin command. I may
> eventually clean it up in the future.
>
> 2. making the parser return a tuple. At kernel_abi.py, as the parser
> returns content from multiple files, such tuple is:
>
> (rst_output, filename, line_number)
>
> and the code for it is (cleaned up):
>
> for msg, f, ln in kernel_abi.doc(show_file=show_file,
> show_symbols=show_symbols,
> filter_path=abi_type):
>
> lines = statemachine.string2lines(msg, tab_width,
> convert_whitespace=True)
>
> for line in lines:
> content.append(line, f, ln - 1) # sphinx counts lines from 0
>
> (2) is cleaner and faster, but (1) is easier to implement on an
> already-existing code.
The logic below implements (1). This seems to be the easiest way for
pyyaml. I will submit as 2 separate patches at the end of the next
version.
Please notice that I didn't check yet for the "quality" of the
line numbers. Some tweaks could be needed later on.
Regards,
Mauro
---
From 750daebebadcd156b5fe9b516f4fae4bd42b9d2c Mon Sep 17 00:00:00 2001
From: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
Date: Tue, 17 Jun 2025 17:54:03 +0200
Subject: [PATCH] docs: parser_yaml.py: add support for line numbers from the
parser
Instead of printing line numbers from the temp converted ReST
file, get them from the original source.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
diff --git a/Documentation/sphinx/parser_yaml.py b/Documentation/sphinx/parser_yaml.py
index 635945e1c5ba..15c642fc0bd5 100755
--- a/Documentation/sphinx/parser_yaml.py
+++ b/Documentation/sphinx/parser_yaml.py
@@ -29,6 +29,8 @@ class YamlParser(Parser):
netlink_parser = YnlDocGenerator()
+ re_lineno = re.compile(r"\.\. LINENO ([0-9]+)$")
+
def do_parse(self, inputstring, document, msg):
"""Parse YAML and generate a document tree."""
@@ -38,8 +40,14 @@ class YamlParser(Parser):
try:
# Parse message with RSTParser
- for i, line in enumerate(msg.split('\n')):
- result.append(line, document.current_source, i)
+ lineoffset = 0;
+ for line in msg.split('\n'):
+ match = self.re_lineno.match(line)
+ if match:
+ lineoffset = int(match.group(1))
+ continue
+
+ result.append(line, document.current_source, lineoffset)
rst_parser = RSTParser()
rst_parser.parse('\n'.join(result), document)
From 15c1f9db30f3abdce110e19788d87f9fe1417781 Mon Sep 17 00:00:00 2001
From: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
Date: Tue, 17 Jun 2025 17:28:04 +0200
Subject: [PATCH] tools: netlink_yml_parser.py: add line numbers to parsed data
When something goes wrong, we want Sphinx error to point to the
right line number from the original source, not from the
processed ReST data.
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
diff --git a/tools/net/ynl/pyynl/netlink_yml_parser.py b/tools/net/ynl/pyynl/netlink_yml_parser.py
index 866551726723..a9d8ab6f2639 100755
--- a/tools/net/ynl/pyynl/netlink_yml_parser.py
+++ b/tools/net/ynl/pyynl/netlink_yml_parser.py
@@ -20,6 +20,16 @@
from typing import Any, Dict, List
import yaml
+LINE_STR = '__lineno__'
+
+class NumberedSafeLoader(yaml.SafeLoader):
+ """Override the SafeLoader class to add line number to parsed data"""
+
+ def construct_mapping(self, node):
+ mapping = super().construct_mapping(node)
+ mapping[LINE_STR] = node.start_mark.line
+
+ return mapping
class RstFormatters:
"""RST Formatters"""
@@ -127,6 +137,11 @@ class RstFormatters:
"""Return a formatted label"""
return f".. _{title}:\n\n"
+ @staticmethod
+ def rst_lineno(lineno: int) -> str:
+ """Return a lineno comment"""
+ return f".. LINENO {lineno}\n"
+
class YnlDocGenerator:
"""YAML Netlink specs Parser"""
@@ -144,6 +159,9 @@ class YnlDocGenerator:
"""Parse 'do' section and return a formatted string"""
lines = []
for key in do_dict.keys():
+ if key == LINE_STR:
+ lines.append(self.fmt.rst_lineno(do_dict[key]))
+ continue
lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1))
if key in ['request', 'reply']:
lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n")
@@ -174,6 +192,10 @@ class YnlDocGenerator:
lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n")
for key in operation.keys():
+ if key == LINE_STR:
+ lines.append(self.fmt.rst_lineno(operation[key]))
+ continue
+
if key in preprocessed:
# Skip the special fields
continue
@@ -233,6 +255,9 @@ class YnlDocGenerator:
for definition in defs:
lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"]))
for k in definition.keys():
+ if k == LINE_STR:
+ lines.append(self.fmt.rst_lineno(definition[k]))
+ continue
if k in preprocessed + ignored:
continue
lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0))
@@ -268,6 +293,9 @@ class YnlDocGenerator:
lines.append(self.fmt.rst_subsubsection(attr_line))
for k in attr.keys():
+ if k == LINE_STR:
+ lines.append(self.fmt.rst_lineno(attr[k]))
+ continue
if k in preprocessed + ignored:
continue
if k in linkable:
@@ -306,6 +334,8 @@ class YnlDocGenerator:
lines = []
# Main header
+ lineno = obj.get('__lineno__', 0)
+ lines.append(self.fmt.rst_lineno(lineno))
family = obj['name']
@@ -354,7 +384,7 @@ class YnlDocGenerator:
def parse_yaml_file(self, filename: str) -> str:
"""Transform the YAML specified by filename into an RST-formatted string"""
with open(filename, "r", encoding="utf-8") as spec_file:
- yaml_data = yaml.safe_load(spec_file)
- content = self.parse_yaml(yaml_data)
+ numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader)
+ content = self.parse_yaml(numbered_yaml)
return content
Powered by blists - more mailing lists