| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20150415194531.GA1659@p183.telecom.by>
Date: Wed, 15 Apr 2015 22:45:31 +0300
From: Alexey Dobriyan <adobriyan@...il.com>
To: Michal Marek <mmarek@...e.cz>
Cc: akpm@...ux-foundation.org, linux-kernel@...r.kernel.org
Subject: [PATCH v2] tags: much faster, parallel "make tags"
On Wed, Apr 15, 2015 at 03:24:26PM +0200, Michal Marek wrote:
> On 2015-04-14 19:20, Alexey Dobriyan wrote:
> > ctags is single-threaded program. Split list of files to be tagged into
> > equal parts, 1 part for each CPU and then merge the results.
> >
> > Speedup on one 2-way box I have is ~143 s => ~99 s (-31%).
> > On another 4-way box: ~120 s => ~65 s (-46%!).
> >
> > Resulting "tags" files aren't byte-for-byte identical because ctags
> > program numbers anon struct and enum declarations with "__anonNNN"
> > symbols. If those lines are removed, "tags" file becomes byte-for-byte
> > identical with those generated with current code.
> >
> > Signed-off-by: Alexey Dobriyan <adobriyan@...il.com>
> > ---
> >
> > scripts/tags.sh | 34 ++++++++++++++++++++++++++++++++--
> > 1 file changed, 32 insertions(+), 2 deletions(-)
> >
> > --- a/scripts/tags.sh
> > +++ b/scripts/tags.sh
> > @@ -152,7 +152,24 @@ dogtags()
> >
> > exuberant()
> > {
> > - all_target_sources | xargs $1 -a \
> > + NR_CPUS=1
> > + if [ -e /proc/cpuinfo ]; then
> > + NR_CPUS=$(grep -e '^processor : ' /proc/cpuinfo | wc -l)
> > + fi
>
> I wonder if we should rather respect the -j option to make here. But
> then most people probably won't realize that make tags is parallel and
> will not use -j when generating tags. So let's leave it as is.
>
>
> > +
> > + rm -f .make-tags.src.* .make-tags.*
>
> .make-tags.src.* is a subset of .make-tags.*
>
>
> > +
> > + all_target_sources >.make-tags.src
> > + # seems like Useless Use of cat(1) but not really
> > + NR_LINES=$(cat .make-tags.src | wc -l)
> > + NR_LINES=$((($NR_LINES + $NR_CPUS - 1) / $NR_CPUS))
> > +
> > + split -a 6 -d -l $NR_LINES .make-tags.src .make-tags.src.
> > +
> > + for i in .make-tags.src.*; do
> > + N=$(echo $i | sed -e 's/.*\.//')
> > + # -u: don't sort now, sort later
> > + cat $i | xargs $1 -a -f .make-tags.$N -u \
>
> xargs <$i $1 ... if you are concerned about uses of cat(1) ;) and the -a
> option is not necessary since we are creating the tmp files.
If "-a" is omitted, tags file becomes broken for some reason,
e. g. "file_operations" isn't there anymore.
> > + # write header
> > + $1 -f tags /dev/null
> > + # remove header
> > + for i in .make-tags.*; do
> > + sed -i -e '/^!/d' $i
> > + done
> > + sort .make-tags.* >>tags
>
> The hardcoded "tags" filename will break 'make TAGS' when using
> exuberant ctags via an 'etags' symlink.
I don't understand why would you fake etags with ctags.
It would go via "emacs" path, which is not changed.
----------------------------------------------------------------------
[PATCH] tags: much faster, parallel "make tags"
ctags is single-threaded program. Split list of files to be tagged into
equal parts, 1 part for each CPU and then merge the results.
Speedup on one 2-way box I have is ~143 s => ~99 s (-31%).
On another 4-way box: ~120 s => ~65 s (-46%!).
Resulting "tags" files aren't byte-for-byte identical because ctags
program numbers anon struct and enum declarations with "__anonNNN"
symbols. If those lines are removed, "tags" file becomes byte-for-byte
identical with those generated with current code.
Signed-off-by: Alexey Dobriyan <adobriyan@...il.com>
---
scripts/tags.sh | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
--- a/scripts/tags.sh
+++ b/scripts/tags.sh
@@ -152,7 +152,19 @@ dogtags()
exuberant()
{
- all_target_sources | xargs $1 -a \
+ rm -f .make-tags.*
+
+ all_target_sources >.make-tags.src
+ NR_CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
+ NR_LINES=$(wc -l <.make-tags.src)
+ NR_LINES=$((($NR_LINES + $NR_CPUS - 1) / $NR_CPUS))
+
+ split -a 6 -d -l $NR_LINES .make-tags.src .make-tags.src.
+
+ for i in .make-tags.src.*; do
+ N=$(echo $i | sed -e 's/.*\.//')
+ # -u: don't sort now, sort later
+ xargs <$i $1 -a -f .make-tags.$N -u \
-I __initdata,__exitdata,__initconst, \
-I __cpuinitdata,__initdata_memblock \
-I __refdata,__attribute,__maybe_unused,__always_unused \
@@ -211,7 +223,21 @@ exuberant()
--regex-c='/DEFINE_PCI_DEVICE_TABLE\((\w*)/\1/v/' \
--regex-c='/(^\s)OFFSET\((\w*)/\2/v/' \
--regex-c='/(^\s)DEFINE\((\w*)/\2/v/' \
- --regex-c='/DEFINE_HASHTABLE\((\w*)/\1/v/'
+ --regex-c='/DEFINE_HASHTABLE\((\w*)/\1/v/' \
+ &
+ done
+ wait
+ rm -f .make-tags.src .make-tags.src.*
+
+ # write header
+ $1 -f $2 /dev/null
+ # remove headers
+ for i in .make-tags.*; do
+ sed -i -e '/^!/d' $i &
+ done
+ wait
+ sort .make-tags.* >>$2
+ rm -f .make-tags.*
all_kconfigs | xargs $1 -a \
--langdef=kconfig --language-force=kconfig \
@@ -276,7 +302,7 @@ emacs()
xtags()
{
if $1 --version 2>&1 | grep -iq exuberant; then
- exuberant $1
+ exuberant $1 $2
elif $1 --version 2>&1 | grep -iq emacs; then
emacs $1
else
@@ -322,13 +348,13 @@ case "$1" in
"tags")
rm -f tags
- xtags ctags
+ xtags ctags tags
remove_structs=y
;;
"TAGS")
rm -f TAGS
- xtags etags
+ xtags etags TAGS
remove_structs=y
;;
esac
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists