Message-ID: <20260122105751.2186609-1-mikko.rapeli@linaro.org>
Date: Thu, 22 Jan 2026 12:57:49 +0200
From: Mikko Rapeli <mikko.rapeli@...aro.org>
To: Nathan Chancellor <nathan@...nel.org>,
	Nicolas Schier <nsc@...nel.org>,
	Anders Roxell <anders.roxell@...aro.org>,
	linux-kbuild@...r.kernel.org,
	linux-kernel@...r.kernel.org
Cc: Mikko Rapeli <mikko.rapeli@...aro.org>
Subject: [PATCH v3 1/3] scripts: kconfig: merge_config.sh: refactor from shell/sed/grep to awk

From: Anders Roxell <anders.roxell@...aro.org>

merge_config.sh shell/sed/grep loop scales poorly and is slow.
With Yocto genericarm64 kernel and around 190 config fragments
the script takes more than 20 minutes to run on a fast build machine.
Re-implementation with awk does the same job in 10 seconds.
Using awk since it is likely available in the build environments
and using perl, python etc would introduce more complex runtime
dependencies. awk is good enough and lot better than shell/sed/grep.

Output stays the same but changed execution time means that
parallel job output may be ordered differently.

Signed-off-by: Anders Roxell <anders.roxell@...aro.org>
Signed-off-by: Mikko Rapeli <mikko.rapeli@...aro.org>
---
 scripts/kconfig/merge_config.sh | 168 ++++++++++++++++++++++++--------
 1 file changed, 128 insertions(+), 40 deletions(-)

v3: use shell variables with plain $VAR syntax, don't convert to =n
    syntax and use " is not set" instead, reverted warning output to
    same as before in the shell script implementation, changed warning
    output from stderr to stdout as in the shell script implementation,
    tested that output from
    make ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config
    and
    make -s ARCH=arm64 CROSS_COMPILE=aarch64-linux- clean defconfig hardening.config
    is the same with and without this change

v2: remove unused sed variables, awk from ${AWK} variable,
    curly brace syntax fix after rebase, triple check that
    correct revision of patches are used in testing with
    yocto/bitbake
    https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#me07dcaaa205b608d56003aeb56a240e5eca1ad52

v1: https://lore.kernel.org/linux-kbuild/20251229114447.45236-1-mikko.rapeli@linaro.org/T/#t

diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index 79c09b378be8..de217771b78c 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -16,8 +16,8 @@
 set -e
 
 clean_up() {
-	rm -f $TMP_FILE
-	rm -f $MERGE_FILE
+	rm -f "$TMP_FILE"
+	rm -f "$TMP_FILE.new"
 }
 
 usage() {
@@ -43,6 +43,10 @@ STRICT=false
 CONFIG_PREFIX=${CONFIG_-CONFIG_}
 WARNOVERRIDE=echo
 
+if [ -z "$AWK" ]; then
+	AWK=awk
+fi
+
 while true; do
 	case $1 in
 	"-n")
@@ -117,11 +121,8 @@ if [ ! -r "$INITFILE" ]; then
 fi
 
 MERGE_LIST=$*
-SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p"
-SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p"
 
 TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
-MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX)
 
 echo "Using $INITFILE as base"
 
@@ -136,42 +137,129 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do
 		echo "The merge file '$ORIG_MERGE_FILE' does not exist.  Exit." >&2
 		exit 1
 	fi
-	cat $ORIG_MERGE_FILE > $MERGE_FILE
-	CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE)
-
-	for CFG in $CFG_LIST ; do
-		grep -q -w $CFG $TMP_FILE || continue
-		PREV_VAL=$(grep -w $CFG $TMP_FILE)
-		NEW_VAL=$(grep -w $CFG $MERGE_FILE)
-		BUILTIN_FLAG=false
-		if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE} -y passed, will not demote y to m
-			${WARNOVERRIDE}
-			BUILTIN_FLAG=true
-		elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then
-			${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE:
-			${WARNOVERRIDE} Previous  value: $PREV_VAL
-			${WARNOVERRIDE} New value:       $NEW_VAL
-			${WARNOVERRIDE}
-			if [ "$STRICT" = "true" ]; then
-				STRICT_MODE_VIOLATED=true
-			fi
-		elif [ "$WARNREDUN" = "true" ]; then
-			${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE:
-		fi
-		if [ "$BUILTIN_FLAG" = "false" ]; then
-			sed -i "/$CFG[ =]/d" $TMP_FILE
-		else
-			sed -i "/$CFG[ =]/d" $MERGE_FILE
-		fi
-	done
-	# In case the previous file lacks a new line at the end
-	echo >> $TMP_FILE
-	cat $MERGE_FILE >> $TMP_FILE
-done
+	# Use awk for single-pass processing instead of per-symbol grep/sed
+	if ! "$AWK" -v prefix="$CONFIG_PREFIX" \
+		-v warnoverride="$WARNOVERRIDE" \
+		-v strict="$STRICT" \
+		-v builtin="$BUILTIN" \
+		-v warnredun="$WARNREDUN" '
+	BEGIN {
+		strict_violated = 0
+		cfg_regex = "^" prefix "[a-zA-Z0-9_]+"
+		notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$"
+	}
+
+	# Extract config name from a line, returns "" if not a config line
+	function get_cfg(line) {
+		if (match(line, cfg_regex)) {
+			return substr(line, RSTART, RLENGTH)
+		} else if (match(line, notset_regex)) {
+			# Extract CONFIG_FOO from "# CONFIG_FOO is not set"
+			sub(/^# /, "", line)
+			sub(/ is not set$/, "", line)
+			return line
+		}
+		return ""
+	}
+
+	function warn_builtin(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print cfg ": -y passed, will not demote y to m"
+		print "Previous value: " prev
+		print "New value: " new
+		print ""
+	}
+
+	function warn_redefined(cfg, prev, new) {
+		if (warnoverride == "true") return
+		print "Value of " cfg " is redefined by fragment " mergefile ":"
+		print "Previous value: " prev
+		print "New value: " new
+		print ""
+	}
+
+	function warn_redundant(cfg) {
+		if (warnredun != "true" || warnoverride == "true") return
+		print "Value of " cfg " is redundant by fragment " mergefile ":"
+	}
+
+	# First pass: read merge file, store all lines and index
+	FILENAME == ARGV[1] {
+	        mergefile = FILENAME
+		merge_lines[FNR] = $0
+		merge_total = FNR
+		cfg = get_cfg($0)
+		if (cfg != "") {
+			merge_cfg[cfg] = $0
+			merge_cfg_line[cfg] = FNR
+		}
+		next
+	}
 
+	# Second pass: process base file (TMP_FILE)
+	FILENAME == ARGV[2] {
+		cfg = get_cfg($0)
+
+		# Not a config or not in merge file - keep it
+		if (cfg == "" || !(cfg in merge_cfg)) {
+			print $0 >> ARGV[3]
+			next
+		}
+
+	        prev_val = $0
+		new_val = merge_cfg[cfg]
+
+		# BUILTIN: do not demote y to m
+		if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) {
+			warn_builtin(cfg, prev_val, new_val)
+			print $0 >> ARGV[3]
+			skip_merge[merge_cfg_line[cfg]] = 1
+			next
+		}
+
+		# Values equal - redundant
+		if (prev_val == new_val) {
+			warn_redundant(cfg)
+			next
+		}
+
+		# "=n" is the same as "is not set"
+		if (prev_val ~ /=n$/ && new_val ~ / is not set$/) {
+			print $0 >> ARGV[3]
+			next
+		}
+
+		# Values differ - redefined
+		warn_redefined(cfg, prev_val, new_val)
+		if (strict == "true") {
+			strict_violated = 1
+		}
+	}
+
+	# output file, skip all lines
+	FILENAME == ARGV[3] {
+		nextfile
+	}
+
+	END {
+		# Newline in case base file lacks trailing newline
+		print "" >> ARGV[3]
+		# Append merge file, skipping lines marked for builtin preservation
+		for (i = 1; i <= merge_total; i++) {
+			if (!(i in skip_merge)) {
+				print merge_lines[i] >> ARGV[3]
+			}
+		}
+		if (strict_violated) {
+			exit 1
+		}
+	}' \
+	"$ORIG_MERGE_FILE" "$TMP_FILE" "$TMP_FILE.new"; then
+		# awk exited non-zero, strict mode was violated
+		STRICT_MODE_VIOLATED=true
+	fi
+	mv "$TMP_FILE.new" "$TMP_FILE"
+done
 if [ "$STRICT_MODE_VIOLATED" = "true" ]; then
 	echo "The fragment redefined a value and strict mode had been passed."
 	exit 1
-- 
2.34.1

Powered by blists - more mailing lists