Message-Id: <20160827152531.66910-1-valentinrothberg@gmail.com>
Date:   Sat, 27 Aug 2016 17:25:29 +0200
From:   Valentin Rothberg <valentinrothberg@...il.com>
To:     gregkh@...uxfoundation.org
Cc:     linux-kernel@...r.kernel.org,
        Valentin Rothberg <valentinrothberg@...il.com>
Subject: [PATCH 1/3] checkkconfigsymbols: port to python3

Port the script to python3 and fix some pylint warnings.

Signed-off-by: Valentin Rothberg <valentinrothberg@...il.com>
---
 scripts/checkkconfigsymbols.py | 280 +++++++++++++++++++++++------------------
 1 file changed, 156 insertions(+), 124 deletions(-)

diff --git a/scripts/checkkconfigsymbols.py b/scripts/checkkconfigsymbols.py
index 0cae73b5c925..9df545615b8d 100755
--- a/scripts/checkkconfigsymbols.py
+++ b/scripts/checkkconfigsymbols.py
@@ -1,13 +1,15 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
-"""Find Kconfig symbols that are referenced but not defined."""
+"""
+Find Kconfig symbols that are referenced but not defined.
+"""
 
 # (c) 2014-2016 Valentin Rothberg <valentinrothberg@...il.com>
 # (c) 2014 Stefan Hengelein <stefan.hengelein@....de>
 #
 # Licensed under the terms of the GNU GPL License version 2
 
-
+import argparse
 import difflib
 import os
 import re
@@ -15,8 +17,6 @@ import signal
 import subprocess
 import sys
 from multiprocessing import Pool, cpu_count
-from optparse import OptionParser
-from subprocess import Popen, PIPE, STDOUT
 
 
 # regex expressions
@@ -42,63 +42,63 @@ REGEX_QUOTES = re.compile("(\"(.*?)\")")
 
 
 def parse_options():
-    """The user interface of this module."""
-    usage = "%prog [options]\n\n"                                              \
-            "Run this tool to detect Kconfig symbols that are referenced but " \
-            "not defined in\nKconfig.  The output of this tool has the "       \
-            "format \'Undefined symbol\\tFile list\'\n\n"                      \
-            "If no option is specified, %prog will default to check your\n"    \
-            "current tree.  Please note that specifying commits will "         \
-            "\'git reset --hard\'\nyour current tree!  You may save "          \
-            "uncommitted changes to avoid losing data."
-
-    parser = OptionParser(usage=usage)
-
-    parser.add_option('-c', '--commit', dest='commit', action='store',
-                      default="",
-                      help="Check if the specified commit (hash) introduces "
-                           "undefined Kconfig symbols.")
-
-    parser.add_option('-d', '--diff', dest='diff', action='store',
-                      default="",
-                      help="Diff undefined symbols between two commits.  The "
-                           "input format bases on Git log's "
-                           "\'commmit1..commit2\'.")
-
-    parser.add_option('-f', '--find', dest='find', action='store_true',
-                      default=False,
-                      help="Find and show commits that may cause symbols to be "
-                           "missing.  Required to run with --diff.")
-
-    parser.add_option('-i', '--ignore', dest='ignore', action='store',
-                      default="",
-                      help="Ignore files matching this pattern.  Note that "
-                           "the pattern needs to be a Python regex.  To "
-                           "ignore defconfigs, specify -i '.*defconfig'.")
-
-    parser.add_option('-s', '--sim', dest='sim', action='store', default="",
-                      help="Print a list of maximum 10 string-similar symbols.")
-
-    parser.add_option('', '--force', dest='force', action='store_true',
-                      default=False,
-                      help="Reset current Git tree even when it's dirty.")
-
-    parser.add_option('', '--no-color', dest='color', action='store_false',
-                      default=True,
-                      help="Don't print colored output. Default when not "
-                           "outputting to a terminal.")
-
-    (opts, _) = parser.parse_args()
-
-    if opts.commit and opts.diff:
+    """
+    The user interface of this module.
+    """
+    usage = "Run this tool to detect Kconfig symbols that are referenced but " \
+            "not defined in Kconfig.  If no option is specified, "             \
+            "checkkconfigsymbols defaults to check your current tree.  "       \
+            "Please note that specifying commits will 'git reset --hard\' "    \
+            "your current tree!  You may save uncommitted changes to avoid "   \
+            "losing data."
+
+    parser = argparse.ArgumentParser(description=usage)
+
+    parser.add_argument('-c', '--commit', dest='commit', action='store',
+                        default="",
+                        help="Check if the specified commit (hash) introduces "
+                             "undefined Kconfig symbols.")
+
+    parser.add_argument('-d', '--diff', dest='diff', action='store',
+                        default="",
+                        help="Diff undefined symbols between two commits.  The "
+                             "input format bases on Git log's "
+                             "\'commmit1..commit2\'.")
+
+    parser.add_argument('-f', '--find', dest='find', action='store_true',
+                        default=False,
+                        help="Find and show commits that may cause symbols to be "
+                             "missing.  Required to run with --diff.")
+
+    parser.add_argument('-i', '--ignore', dest='ignore', action='store',
+                        default="",
+                        help="Ignore files matching this pattern.  Note that "
+                             "the pattern needs to be a Python regex.  To "
+                             "ignore defconfigs, specify -i '.*defconfig'.")
+
+    parser.add_argument('-s', '--sim', dest='sim', action='store', default="",
+                        help="Print a list of maximum 10 string-similar symbols.")
+
+    parser.add_argument('--force', dest='force', action='store_true',
+                        default=False,
+                        help="Reset current Git tree even when it's dirty.")
+
+    parser.add_argument('--no-color', dest='color', action='store_false',
+                        default=True,
+                        help="Don't print colored output. Default when not "
+                             "outputting to a terminal.")
+
+    args = parser.parse_args()
+
+    if args.commit and args.diff:
         sys.exit("Please specify only one option at once.")
 
-    if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
+    if args.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", args.diff):
         sys.exit("Please specify valid input in the following format: "
                  "\'commit1..commit2\'")
 
-    if opts.commit or opts.diff:
-        if not opts.force and tree_is_dirty():
+    if args.commit or args.diff:
+        if not args.force and tree_is_dirty():
             sys.exit("The current Git tree is dirty (see 'git status').  "
                      "Running this script may\ndelete important data since it "
                      "calls 'git reset --hard' for some performance\nreasons. "
@@ -106,48 +106,50 @@ def parse_options():
                      "'--force' if you\nwant to ignore this warning and "
                      "continue.")
 
-    if opts.commit:
-        opts.find = False
+    if args.commit:
+        args.find = False
 
-    if opts.ignore:
+    if args.ignore:
         try:
-            re.match(opts.ignore, "this/is/just/a/test.c")
+            re.match(args.ignore, "this/is/just/a/test.c")
         except:
             sys.exit("Please specify a valid Python regex.")
 
-    return opts
+    return args
 
 
 def main():
-    """Main function of this module."""
-    opts = parse_options()
+    """
+    Main function of this module.
+    """
+    args = parse_options()
 
-    global color
-    color = opts.color and sys.stdout.isatty()
+    global COLOR
+    COLOR = args.color and sys.stdout.isatty()
 
-    if opts.sim and not opts.commit and not opts.diff:
-        sims = find_sims(opts.sim, opts.ignore)
+    if args.sim and not args.commit and not args.diff:
+        sims = find_sims(args.sim, args.ignore)
         if sims:
-            print "%s: %s" % (yel("Similar symbols"), ', '.join(sims))
+            print("%s: %s" % (yel("Similar symbols"), ', '.join(sims)))
         else:
-            print "%s: no similar symbols found" % yel("Similar symbols")
+            print("%s: no similar symbols found" % yel("Similar symbols"))
         sys.exit(0)
 
     # dictionary of (un)defined symbols
     defined = {}
     undefined = {}
 
-    if opts.commit or opts.diff:
+    if args.commit or args.diff:
         head = get_head()
 
         # get commit range
         commit_a = None
         commit_b = None
-        if opts.commit:
-            commit_a = opts.commit + "~"
-            commit_b = opts.commit
-        elif opts.diff:
-            split = opts.diff.split("..")
+        if args.commit:
+            commit_a = args.commit + "~"
+            commit_b = args.commit
+        elif args.diff:
+            split = args.diff.split("..")
             commit_a = split[0]
             commit_b = split[1]
             undefined_a = {}
@@ -155,16 +157,16 @@ def main():
 
         # get undefined items before the commit
         execute("git reset --hard %s" % commit_a)
-        undefined_a, _ = check_symbols(opts.ignore)
+        undefined_a, _ = check_symbols(args.ignore)
 
         # get undefined items for the commit
         execute("git reset --hard %s" % commit_b)
-        undefined_b, defined = check_symbols(opts.ignore)
+        undefined_b, defined = check_symbols(args.ignore)
 
         # report cases that are present for the commit but not before
         for feature in sorted(undefined_b):
             # feature has not been undefined before
-            if not feature in undefined_a:
+            if feature not in undefined_a:
                 files = sorted(undefined_b.get(feature))
                 undefined[feature] = files
             # check if there are new files that reference the undefined feature
@@ -179,68 +181,75 @@ def main():
 
     # default to check the entire tree
     else:
-        undefined, defined = check_symbols(opts.ignore)
+        undefined, defined = check_symbols(args.ignore)
 
     # now print the output
     for feature in sorted(undefined):
-        print red(feature)
+        print(red(feature))
 
         files = sorted(undefined.get(feature))
-        print "%s: %s" % (yel("Referencing files"), ", ".join(files))
+        print("%s: %s" % (yel("Referencing files"), ", ".join(files)))
 
-        sims = find_sims(feature, opts.ignore, defined)
+        sims = find_sims(feature, args.ignore, defined)
         sims_out = yel("Similar symbols")
         if sims:
-            print "%s: %s" % (sims_out, ', '.join(sims))
+            print("%s: %s" % (sims_out, ', '.join(sims)))
         else:
-            print "%s: %s" % (sims_out, "no similar symbols found")
+            print("%s: %s" % (sims_out, "no similar symbols found"))
 
-        if opts.find:
-            print "%s:" % yel("Commits changing symbol")
-            commits = find_commits(feature, opts.diff)
+        if args.find:
+            print("%s:" % yel("Commits changing symbol"))
+            commits = find_commits(feature, args.diff)
             if commits:
                 for commit in commits:
                     commit = commit.split(" ", 1)
-                    print "\t- %s (\"%s\")" % (yel(commit[0]), commit[1])
+                    print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1]))
             else:
-                print "\t- no commit found"
-        print  #  new line
+                print("\t- no commit found")
+        print()  # new line
 
 
 def yel(string):
     """
     Color %string yellow.
     """
-    return "\033[33m%s\033[0m" % string if color else string
+    return "\033[33m%s\033[0m" % string if COLOR else string
 
 
 def red(string):
     """
     Color %string red.
     """
-    return "\033[31m%s\033[0m" % string if color else string
+    return "\033[31m%s\033[0m" % string if COLOR else string
 
 
 def execute(cmd):
-    """Execute %cmd and return stdout.  Exit in case of error."""
+    """
+    Execute %cmd and return stdout.  Exit in case of error.
+    """
     try:
         cmdlist = cmd.split(" ")
-        stdout = subprocess.check_output(cmdlist, stderr=STDOUT, shell=False)
+        stdout = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT, shell=False)
+        stdout = stdout.decode(errors='replace')
     except subprocess.CalledProcessError as fail:
         exit("Failed to execute %s\n%s" % (cmd, fail))
     return stdout
 
 
 def find_commits(symbol, diff):
-    """Find commits changing %symbol in the given range of %diff."""
+    """
+    Find commits changing %symbol in the given range of %diff.
+    """
     commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s"
                       % (symbol, diff))
     return [x for x in commits.split("\n") if x]
 
 
 def tree_is_dirty():
-    """Return true if the current working tree is dirty (i.e., if any file has
-    been added, deleted, modified, renamed or copied but not committed)."""
+    """
+    Return true if the current working tree is dirty (i.e., if any file has been
+    added, deleted, modified, renamed or copied but not committed).
+    """
     stdout = execute("git status --porcelain")
     for line in stdout:
         if re.findall(r"[URMADC]{1}", line[:2]):
@@ -249,24 +258,32 @@ def tree_is_dirty():
 
 
 def get_head():
-    """Return commit hash of current HEAD."""
+    """
+    Return commit hash of current HEAD.
+    """
     stdout = execute("git rev-parse HEAD")
     return stdout.strip('\n')
 
 
 def partition(lst, size):
-    """Partition list @lst into eveni-sized lists of size @size."""
-    return [lst[i::size] for i in xrange(size)]
+    """
+    Partition list @lst into eveni-sized lists of size @size.
+    """
+    return [lst[i::size] for i in range(size)]
 
 
 def init_worker():
-    """Set signal handler to ignore SIGINT."""
+    """
+    Set signal handler to ignore SIGINT.
+    """
     signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 
-def find_sims(symbol, ignore, defined = []):
-    """Return a list of max. ten Kconfig symbols that are string-similar to
-    @symbol."""
+def find_sims(symbol, ignore, defined=[]):
+    """
+    Return a list of max. ten Kconfig symbols that are string-similar to
+    @symbol.
+    """
     if defined:
         return sorted(difflib.get_close_matches(symbol, set(defined), 10))
 
@@ -287,7 +304,9 @@ def find_sims(symbol, ignore, defined = []):
 
 
 def get_files():
-    """Return a list of all files in the current git directory."""
+    """
+    Return a list of all files in the current git directory.
+    """
     # use 'git ls-files' to get the worklist
     stdout = execute("git ls-files")
     if len(stdout) > 0 and stdout[-1] == "\n":
@@ -304,9 +323,11 @@ def get_files():
 
 
 def check_symbols(ignore):
-    """Find undefined Kconfig symbols and return a dict with the symbol as key
-    and a list of referencing files as value.  Files matching %ignore are not
-    checked for undefined symbols."""
+    """
+    Find undefined Kconfig symbols and return a dict with the symbol as key and
+    a list of referencing files as value.  Files matching %ignore are not
+    checked for undefined symbols.
+    """
     pool = Pool(cpu_count(), init_worker)
     try:
         return check_symbols_helper(pool, ignore)
@@ -317,8 +338,10 @@ def check_symbols(ignore):
 
 
 def check_symbols_helper(pool, ignore):
-    """Helper method for check_symbols().  Used to catch keyboard interrupts in
-    check_symbols() in order to properly terminate running worker processes."""
+    """
+    Helper method for check_symbols().  Used to catch keyboard interrupts in
+    check_symbols() in order to properly terminate running worker processes.
+    """
     source_files = []
     kconfig_files = []
     defined_features = []
@@ -338,7 +361,6 @@ def check_symbols_helper(pool, ignore):
     for res in pool.map(parse_source_files, arglist):
         referenced_features.update(res)
 
-
     # parse kconfig files
     arglist = []
     for part in partition(kconfig_files, cpu_count()):
@@ -350,7 +372,7 @@ def check_symbols_helper(pool, ignore):
 
     # inverse mapping of referenced_features to dict(feature: [files])
     inv_map = dict()
-    for _file, features in referenced_features.iteritems():
+    for _file, features in referenced_features.items():
         for feature in features:
             inv_map[feature] = inv_map.get(feature, set())
             inv_map[feature].add(_file)
@@ -372,8 +394,10 @@ def check_symbols_helper(pool, ignore):
 
 
 def parse_source_files(source_files):
-    """Parse each source file in @source_files and return dictionary with source
-    files as keys and lists of references Kconfig symbols as values."""
+    """
+    Parse each source file in @source_files and return dictionary with source
+    files as keys and lists of references Kconfig symbols as values.
+    """
     referenced_features = dict()
     for sfile in source_files:
         referenced_features[sfile] = parse_source_file(sfile)
@@ -381,18 +405,20 @@ def parse_source_files(source_files):
 
 
 def parse_source_file(sfile):
-    """Parse @sfile and return a list of referenced Kconfig features."""
+    """
+    Parse @sfile and return a list of referenced Kconfig features.
+    """
     lines = []
     references = []
 
     if not os.path.exists(sfile):
         return references
 
-    with open(sfile, "r") as stream:
+    with open(sfile, "r", encoding='utf-8', errors='replace') as stream:
         lines = stream.readlines()
 
     for line in lines:
-        if not "CONFIG_" in line:
+        if "CONFIG_" not in line:
             continue
         features = REGEX_SOURCE_FEATURE.findall(line)
         for feature in features:
@@ -404,14 +430,17 @@ def parse_source_file(sfile):
 
 
 def get_features_in_line(line):
-    """Return mentioned Kconfig features in @line."""
+    """
+    Return mentioned Kconfig features in @line.
+    """
     return REGEX_FEATURE.findall(line)
 
 
 def parse_kconfig_files(args):
-    """Parse kconfig files and return tuple of defined and references Kconfig
-    symbols.  Note, @args is a tuple of a list of files and the @ignore
-    pattern."""
+    """
+    Parse kconfig files and return tuple of defined and references Kconfig
+    symbols.  Note, @args is a tuple of a list of files and the @ignore pattern.
+    """
     kconfig_files = args[0]
     ignore = args[1]
     defined_features = []
@@ -428,7 +457,9 @@ def parse_kconfig_files(args):
 
 
 def parse_kconfig_file(kfile):
-    """Parse @kfile and update feature definitions and references."""
+    """
+    Parse @kfile and update feature definitions and references.
+    """
     lines = []
     defined = []
     references = []
@@ -437,7 +468,7 @@ def parse_kconfig_file(kfile):
     if not os.path.exists(kfile):
         return defined, references
 
-    with open(kfile, "r") as stream:
+    with open(kfile, "r", encoding='utf-8', errors='replace') as stream:
         lines = stream.readlines()
 
     for i in range(len(lines)):
@@ -473,4 +504,5 @@ def parse_kconfig_file(kfile):
 
 
 if __name__ == "__main__":
+    COLOR = None
     main()
-- 
2.9.3

Powered by blists - more mailing lists