linux-kernel - [PATCH v4] tags: much faster, parallel "make tags"

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <20150511202515.GB7650@p183.telecom.by>
Date:	Mon, 11 May 2015 23:25:15 +0300
From:	Alexey Dobriyan <adobriyan@...il.com>
To:	Andrew Morton <akpm@...ux-foundation.org>
Cc:	Michal Marek <mmarek@...e.cz>, linux-kernel@...r.kernel.org,
	Pádraig Brady <P@...igBrady.com>
Subject: [PATCH v4] tags: much faster, parallel "make tags"

ctags is single-threaded program. Split list of files to be tagged into
almost equal parts, process them on every CPU and merge the results.

Speedup is ~30-45% (!) (depending on number of cores).

Resulting "tags" files aren't byte-for-byte identical because ctags
program numbers anon struct and enum declarations with "__anonNNN"
symbols. If those lines are removed, "tags" file becomes byte-for-byte
identical with those generated with current code.

v4: switch from shell "&; wait"' parallelism to "xargs -P" for reliable cleanup.

Signed-off-by: Alexey Dobriyan <adobriyan@...il.com>
---

 scripts/tags.sh |   58 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 53 insertions(+), 5 deletions(-)

--- a/scripts/tags.sh
+++ b/scripts/tags.sh
@@ -152,7 +152,41 @@ dogtags()
 
 exuberant()
 {
-	all_target_sources | xargs $1 -a                        \
+	trap 'rm -f .make-tags.*; exit 1' TERM INT
+	rm -f .make-tags.*
+
+	all_target_sources >.make-tags.0
+
+	# Default xargs(1) total command line size.
+	XARGS_ARG_MAX=$((128 * 1024))
+	# Split is unequal w.r.t file count, but asking for both size and
+	# line count limit is too much in 2015.
+	#
+	# Reserve room for fixed ctags(1) arguments.
+	split -a 6 -d -C $(($XARGS_ARG_MAX - 4 * 1024)) .make-tags.0 .make-tags.x
+	rm -f .make-tags.0
+
+	# xargs(1) appears to not support command line tweaking,
+	# so it has to be prepared in advance (see '-f').
+	NR_TAGS=$(ls -1 .make-tags.x* | wc -l)
+	touch .make-tags.1
+	for i in $(seq 0 $(($NR_TAGS - 1))); do
+		N=$(printf "%06u" $i)
+		echo -n "-f .make-tags.t$N " >>.make-tags.1
+		tr '\n' ' ' <.make-tags.x$N >>.make-tags.1
+		echo >>.make-tags.1
+		rm -f .make-tags.x$N
+	done
+
+	# Tag files in parallel.
+	#
+	# "xargs -I" puts command line piece as one argument,
+	# so shell is employed to split it back.
+	NR_CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
+	# ctags -u: don't sort now, sort later
+	xargs -P $NR_CPUS -L 1 -I CMD -s $XARGS_ARG_MAX		\
+		<.make-tags.1					\
+		sh -c "$1 -a -u					\
 	-I __initdata,__exitdata,__initconst,			\
 	-I __cpuinitdata,__initdata_memblock			\
 	-I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -211,7 +245,21 @@ exuberant()
 	--regex-c='/DEFINE_PCI_DEVICE_TABLE\((\w*)/\1/v/'		\
 	--regex-c='/(^\s)OFFSET\((\w*)/\2/v/'				\
 	--regex-c='/(^\s)DEFINE\((\w*)/\2/v/'				\
-	--regex-c='/DEFINE_HASHTABLE\((\w*)/\1/v/'
+	--regex-c='/DEFINE_HASHTABLE\((\w*)/\1/v/'			\
+	CMD"
+	rm -f .make-tags.1
+
+	# Remove headers.
+	for i in .make-tags.t*; do
+		sed -i -e '/^!/d' $i
+	done
+
+	# Write final header.
+	$1 -f $2 /dev/null
+
+	# Append sorted results.
+	sort .make-tags.t* >>$2
+	rm -f .make-tags.t*
 
 	all_kconfigs | xargs $1 -a                              \
 	--langdef=kconfig --language-force=kconfig              \
@@ -276,7 +324,7 @@ emacs()
 xtags()
 {
 	if $1 --version 2>&1 | grep -iq exuberant; then
-		exuberant $1
+		exuberant $1 $2
 	elif $1 --version 2>&1 | grep -iq emacs; then
 		emacs $1
 	else
@@ -322,13 +370,13 @@ case "$1" in
 
 	"tags")
 		rm -f tags
-		xtags ctags
+		xtags ctags tags
 		remove_structs=y
 		;;
 
 	"TAGS")
 		rm -f TAGS
-		xtags etags
+		xtags etags TAGS
 		remove_structs=y
 		;;
 esac
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/