| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <6964e8bb-d2f1-4203-94db-857135cf7ecf@gmail.com>
Date: Wed, 12 Jun 2024 09:34:19 +0800
From: Alex Shi <seakeel@...il.com>
To: Dongliang Mu <dzm91@...t.edu.cn>, Alex Shi <alexs@...nel.org>,
Yanteng Si <siyanteng@...ngson.cn>, Jonathan Corbet <corbet@....net>
Cc: hust-os-kernel-patches@...glegroups.com, linux-doc@...r.kernel.org,
Cheng Ziqiu <chengziqiu@...t.edu.cn>, linux-kernel@...r.kernel.org
Subject: Re: [PATCH v2] scripts: add scripts/checktransupdate.py
On 6/11/24 9:17 PM, Dongliang Mu wrote:
> The checktransupdate.py script helps track the translation status of
> the documentation in different locales, e.g., zh_CN and verify if
> these documenation is up-to-date. More specially, it uses `git log`
> commit to find the latest english commit from the translation commit
> (order by author date) and the latest english commits from HEAD. If
> differences occur, report the file and commits that need to be updated.
>
> Signed-off-by: Dongliang Mu <dzm91@...t.edu.cn>
> Signed-off-by: Cheng Ziqiu <chengziqiu@...t.edu.cn>
> ---
Reviewed-by: Alex Shi <alexs@...nel.org>
> v1->v2: revise the output format of git commits
> add some description and usage of this script
> scripts/checktransupdate.py | 203 ++++++++++++++++++++++++++++++++++++
> 1 file changed, 203 insertions(+)
> create mode 100755 scripts/checktransupdate.py
>
> diff --git a/scripts/checktransupdate.py b/scripts/checktransupdate.py
> new file mode 100755
> index 000000000000..5a0fc99e3f93
> --- /dev/null
> +++ b/scripts/checktransupdate.py
> @@ -0,0 +1,203 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0
> +
> +"""
> +This script helps track the translation status of the documentation
> +in different locales, e.g., zh_CN. More specially, it uses `git log`
> +commit to find the latest english commit from the translation commit
> +(order by author date) and the latest english commits from HEAD. If
> +differences occur, report the file and commits that need to be updated.
> +
> +The usage is as follows:
> +- ./scripts/checktransupdate.py -l zh_CN
> +This will print all the files that need to be updated in the zh_CN locale.
> +- ./scripts/checktransupdate.py Documentation/translations/zh_CN/dev-tools/testing-overview.rst
> +This will only print the status of the specified file.
> +
> +The output is something like:
> +Documentation/translations/zh_CN/dev-tools/testing-overview.rst (1 commits)
> +commit 42fb9cfd5b18 ("Documentation: dev-tools: Add link to RV docs")
> +"""
> +
> +import os
> +from argparse import ArgumentParser, BooleanOptionalAction
> +from datetime import datetime
> +
> +flag_p_c = False
> +flag_p_uf = False
> +flag_debug = False
> +
> +
> +def dprint(*args, **kwargs):
> + if flag_debug:
> + print("[DEBUG] ", end="")
> + print(*args, **kwargs)
> +
> +
> +def get_origin_path(file_path):
> + paths = file_path.split("/")
> + tidx = paths.index("translations")
> + opaths = paths[:tidx]
> + opaths += paths[tidx + 2 :]
> + return "/".join(opaths)
> +
> +
> +def get_latest_commit_from(file_path, commit):
> + command = "git log --pretty=format:%H%n%aD%n%cD%n%n%B {} -1 -- {}".format(
> + commit, file_path
> + )
> + dprint(command)
> + pipe = os.popen(command)
> + result = pipe.read()
> + result = result.split("\n")
> + if len(result) <= 1:
> + return None
> +
> + dprint("Result: {}".format(result[0]))
> +
> + return {
> + "hash": result[0],
> + "author_date": datetime.strptime(result[1], "%a, %d %b %Y %H:%M:%S %z"),
> + "commit_date": datetime.strptime(result[2], "%a, %d %b %Y %H:%M:%S %z"),
> + "message": result[4:],
> + }
> +
> +
> +def get_origin_from_trans(origin_path, t_from_head):
> + o_from_t = get_latest_commit_from(origin_path, t_from_head["hash"])
> + while o_from_t is not None and o_from_t["author_date"] > t_from_head["author_date"]:
> + o_from_t = get_latest_commit_from(origin_path, o_from_t["hash"] + "^")
> + if o_from_t is not None:
> + dprint("tracked origin commit id: {}".format(o_from_t["hash"]))
> + return o_from_t
> +
> +
> +def get_commits_count_between(opath, commit1, commit2):
> + command = "git log --pretty=format:%H {}...{} -- {}".format(commit1, commit2, opath)
> + dprint(command)
> + pipe = os.popen(command)
> + result = pipe.read().split("\n")
> + # filter out empty lines
> + result = list(filter(lambda x: x != "", result))
> + return result
> +
> +
> +def pretty_output(commit):
> + command = "git log --pretty='format:%h (\"%s\")' -1 {}".format(commit)
> + dprint(command)
> + pipe = os.popen(command)
> + return pipe.read()
> +
> +
> +def check_per_file(file_path):
> + opath = get_origin_path(file_path)
> +
> + if not os.path.isfile(opath):
> + dprint("Error: Cannot find the origin path for {}".format(file_path))
> + return
> +
> + o_from_head = get_latest_commit_from(opath, "HEAD")
> + t_from_head = get_latest_commit_from(file_path, "HEAD")
> +
> + if o_from_head is None or t_from_head is None:
> + print("Error: Cannot find the latest commit for {}".format(file_path))
> + return
> +
> + o_from_t = get_origin_from_trans(opath, t_from_head)
> +
> + if o_from_t is None:
> + print("Error: Cannot find the latest origin commit for {}".format(file_path))
> + return
> +
> + if o_from_head["hash"] == o_from_t["hash"]:
> + if flag_p_uf:
> + print("No update needed for {}".format(file_path))
> + return
> + else:
> + print("{}".format(file_path), end="\t")
> + commits = get_commits_count_between(
> + opath, o_from_t["hash"], o_from_head["hash"]
> + )
> + print("({} commits)".format(len(commits)))
> + if flag_p_c:
> + for commit in commits:
> + msg = pretty_output(commit)
> + if "Merge tag" not in msg:
> + print("commit", msg)
> +
> +
> +def main():
> + script_path = os.path.dirname(os.path.abspath(__file__))
> + linux_path = os.path.join(script_path, "..")
> +
> + parser = ArgumentParser(description="Check the translation update")
> + parser.add_argument(
> + "-l",
> + "--locale",
> + help="Locale to check when files are not specified",
> + )
> + parser.add_argument(
> + "--print-commits",
> + action=BooleanOptionalAction,
> + default=True,
> + help="Print commits between the origin and the translation",
> + )
> +
> + parser.add_argument(
> + "--print-updated-files",
> + action=BooleanOptionalAction,
> + default=False,
> + help="Print files that do no need to be updated",
> + )
> +
> + parser.add_argument(
> + "--debug",
> + action=BooleanOptionalAction,
> + help="Print debug information",
> + default=False,
> + )
> +
> + parser.add_argument(
> + "files", nargs="*", help="Files to check, if not specified, check all files"
> + )
> + args = parser.parse_args()
> +
> + global flag_p_c, flag_p_uf, flag_debug
> + flag_p_c = args.print_commits
> + flag_p_uf = args.print_updated_files
> + flag_debug = args.debug
> +
> + # get files related to linux path
> + files = args.files
> + if len(files) == 0:
> + if args.locale is not None:
> + files = (
> + os.popen(
> + "find {}/Documentation/translations/{} -type f".format(
> + linux_path, args.locale
> + )
> + )
> + .read()
> + .split("\n")
> + )
> + else:
> + files = (
> + os.popen(
> + "find {}/Documentation/translations -type f".format(linux_path)
> + )
> + .read()
> + .split("\n")
> + )
> +
> + files = list(filter(lambda x: x != "", files))
> + files = list(map(lambda x: os.path.relpath(os.path.abspath(x), linux_path), files))
> +
> + # cd to linux root directory
> + os.chdir(linux_path)
> +
> + for file in files:
> + check_per_file(file)
> +
> +
> +if __name__ == "__main__":
> + main()
Powered by blists - more mailing lists