linux-kernel - [PATCH] scripts: check_docs_external

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [day] [month] [year] [list]

Message-Id: <73387e5c0c4fd99c37303dee184ad09e505c81bb.1602152989.git.mchehab+huawei@kernel.org>
Date:   Thu,  8 Oct 2020 12:30:24 +0200
From:   Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
To:     "Jonathan Corbet" <corbet@....net>,
        Linux Doc Mailing List <linux-doc@...r.kernel.org>
Cc:     Mauro Carvalho Chehab <mchehab+huawei@...nel.org>,
        linux-kernel@...r.kernel.org
Subject: [PATCH] scripts: check_docs_external_symbols: speed up its execution

This script is slow. Speed it up by using one thread per CPU.

On my desktop with 4 cores (8 threads) and SSD disks, before
this change, it takes:

	$ time scripts/check_docs_external_symbols drivers/media/v4l2-core/
	...
	real	0m11,044s
	user	0m13,860s
	sys	0m2,048s

After it:

	$ time scripts/check_docs_external_symbols drivers/media/v4l2-core/
	...
	real	0m3,153s
	user	0m19,322s
	sys	0m2,738s

So, it is now almost 4 times faster.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@...nel.org>
---
 scripts/check_docs_external_symbols | 46 +++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/scripts/check_docs_external_symbols b/scripts/check_docs_external_symbols
index e04af5f03a1d..d312e1973530 100755
--- a/scripts/check_docs_external_symbols
+++ b/scripts/check_docs_external_symbols
@@ -23,6 +23,8 @@ use warnings;
 use strict;
 use File::Find;
 use Cwd 'abs_path';
+use threads;
+use Thread::Queue;
 
 sub check_kerneldoc_symbols($$$$) {
 	my $file = shift;
@@ -131,14 +133,15 @@ sub check_kerneldoc_symbols($$$$) {
 	return %hash;
 }
 
-sub check_file($) {
+sub do_check_file($) {
 	my $file = shift;
 	my (@files, @exports, @doc, @doc_refs, %file_exports);
 	my $content = "\n";
 
-	$file =~ s/\s+$//;
-
-	return 0 if (!($file =~ /\.[ch]$/));
+	local $SIG{'KILL'} = sub {
+		print "$$ aborted.\n";
+		exit(1);
+	};
 
 	my $dir = $file;
 	$dir =~ s,[^\/]+$,,;
@@ -262,6 +265,20 @@ sub check_file($) {
 	return 1;
 }
 
+my $queue;
+
+sub check_file($) {
+	my $file = shift;
+
+	$file =~ s/\s+$//;
+
+	return if (!($file =~ /\.[ch]$/));
+
+#printf "queuing $file\n";
+
+	$queue->enqueue($file);
+}
+
 sub parse_dir {
 	check_file $File::Find::name;
 }
@@ -270,6 +287,20 @@ sub parse_dir {
 # main
 #
 
+my $cpus = qx(nproc);
+
+$queue = Thread::Queue->new();
+
+for (my $i = 0; $i < $cpus; $i++) {
+	threads->create(
+		sub {
+			while (defined(my $file = $queue->dequeue())) {
+				do_check_file($file);
+			}
+		}
+	);
+};
+
 if (@ARGV) {
 	while (@ARGV) {
 		my $file = shift;
@@ -280,10 +311,15 @@ if (@ARGV) {
 			check_file $file;
 		}
 	}
-	exit;
 } else {
 	my @files = qx(git grep -l EXPORT_SYMBOL);
 	foreach my $file (@files) {
 		check_file $file;
 	}
 }
+
+$queue->end();
+
+foreach my $thr(threads->list()) {
+	$thr->join();
+}
-- 
2.26.2