Merge branch '2019-10-08-master-imports'

- Python3 conversion of genboardscfg.py
- Resync Kconfiglib.py
- Switch to running CI on Ubuntu "bionic" to facilitate Python 3.6 being
  the minimum we use and test.

.gitlab-ci.yml

@@ -2,7 +2,7 @@
 
 # Grab our configured image.  The source for this is found at:
 # https://gitlab.denx.de/u-boot/gitlab-ci-runner
-image: trini/u-boot-gitlab-ci-runner:xenial-20190720-02Aug2019
+image: trini/u-boot-gitlab-ci-runner:bionic-20190912.1-03Oct2019
 
 # We run some tests in different order, to catch some failures quicker.
 stages:
@@ -22,8 +22,9 @@ stages:
     - . /tmp/venv/bin/activate
     - pip install pytest==2.8.7
     - pip install python-subunit
-    - grub-mkimage -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
+    - pip install coverage
-    - grub-mkimage -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
+    - grub-mkimage --prefix="" -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
+    - grub-mkimage --prefix="" -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
     - mkdir ~/grub2-arm
     - ( cd ~/grub2-arm; wget -O - http://download.opensuse.org/ports/armv7hl/distribution/leap/42.2/repo/oss/suse/armv7hl/grub2-arm-efi-2.02~beta2-87.1.armv7hl.rpm | rpm2cpio | cpio -di )
     - mkdir ~/grub2-arm64
@@ -36,9 +37,9 @@ stages:
     # use clang only do one configuration.
     - if [[ "${BUILDMAN}" != "" ]]; then
         ret=0;
-        tools/buildman/buildman -P -E ${BUILDMAN} ${OVERRIDE}|| ret=$?;
+        tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE}|| ret=$?;
         if [[ $ret -ne 0 && $ret -ne 129 ]]; then
-          tools/buildman/buildman -sdeP ${BUILDMAN};
+          tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN};
           exit $ret;
         fi;
       fi
@@ -46,7 +47,7 @@ stages:
     # never prevent any test from running. That way, we can always pass
     # "-k something" even when $TEST_PY_TEST_SPEC doesnt need a custom
     # value.
-    - export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/.bm-work/${TEST_PY_BD};
+    - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/${TEST_PY_BD};
       export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:/usr/bin:/bin;
       export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci;
       if [[ "${TEST_PY_BD}" != "" ]]; then
@@ -64,9 +65,9 @@ build all 32bit ARM platforms:
   stage: world build
   script:
     - ret=0;
-     ./tools/buildman/buildman -P -E arm -x aarch64 || ret=$?;
+     ./tools/buildman/buildman -o /tmp -P -E arm -x aarch64 || ret=$?;
      if [[ $ret -ne 0 && $ret -ne 129 ]]; then
-       ./tools/buildman/buildman -sdeP;
+       ./tools/buildman/buildman -o /tmp -sdeP;
        exit $ret;
      fi;
 
@@ -78,9 +79,9 @@ build all 64bit ARM platforms:
     - . /tmp/venv/bin/activate
     - pip install pyelftools
     - ret=0;
-     ./tools/buildman/buildman -P -E aarch64 || ret=$?;
+     ./tools/buildman/buildman -o /tmp -P -E aarch64 || ret=$?;
      if [[ $ret -ne 0 && $ret -ne 129 ]]; then
-       ./tools/buildman/buildman -sdeP;
+       ./tools/buildman/buildman -o /tmp -sdeP;
        exit $ret;
      fi;
 
@@ -89,9 +90,9 @@ build all PowerPC platforms:
   stage: world build
   script:
     - ret=0;
-     ./tools/buildman/buildman -P -E powerpc || ret=$?;
+     ./tools/buildman/buildman -o /tmp -P -E powerpc || ret=$?;
      if [[ $ret -ne 0 && $ret -ne 129 ]]; then
-       ./tools/buildman/buildman -sdeP;
+       ./tools/buildman/buildman -o /tmp -sdeP;
        exit $ret;
      fi;
 
@@ -100,9 +101,9 @@ build all other platforms:
   stage: world build
   script:
     - ret=0;
-     ./tools/buildman/buildman -P -E -x arm,powerpc || ret=$?;
+     ./tools/buildman/buildman -o /tmp -P -E -x arm,powerpc || ret=$?;
      if [[ $ret -ne 0 && $ret -ne 129 ]]; then
-       ./tools/buildman/buildman -sdeP;
+       ./tools/buildman/buildman -o /tmp -sdeP;
        exit $ret;
      fi;
 
@@ -162,10 +163,10 @@ Run binman, buildman, dtoc and patman testsuites:
       virtualenv /tmp/venv;
       . /tmp/venv/bin/activate;
       pip install pyelftools;
-      export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/.bm-work/sandbox_spl;
+      export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/sandbox_spl;
       export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt";
       export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}";
-      ./tools/buildman/buildman -P sandbox_spl;
+      ./tools/buildman/buildman -o /tmp -P sandbox_spl;
       ./tools/binman/binman --toolpath ${UBOOT_TRAVIS_BUILD_DIR}/tools test;
       ./tools/buildman/buildman -t;
       ./tools/dtoc/dtoc -t;

.travis.yml

@@ -4,7 +4,7 @@
 # build U-Boot on Travis CI - https://travis-ci.org/
 
 sudo: required
-dist: xenial
+dist: bionic
 
 language: c
 
@@ -12,7 +12,7 @@ addons:
   apt:
     sources:
     - ubuntu-toolchain-r-test
-    - llvm-toolchain-xenial-7
+    - llvm-toolchain-bionic-7
     packages:
     - cppcheck
     - sloccount
@@ -52,12 +52,13 @@ install:
  - pip install pytest==2.8.7
  - pip install python-subunit
  - pip install pyelftools
- - grub-mkimage -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
+ - grub-mkimage --prefix="" -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
- - grub-mkimage -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
+ - grub-mkimage --prefix="" -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
  - mkdir ~/grub2-arm
  - ( cd ~/grub2-arm; wget -O - http://download.opensuse.org/ports/armv7hl/distribution/leap/42.2/repo/oss/suse/armv7hl/grub2-arm-efi-2.02~beta2-87.1.armv7hl.rpm | rpm2cpio | cpio -di )
  - mkdir ~/grub2-arm64
  - ( cd ~/grub2-arm64; wget -O - http://download.opensuse.org/ports/aarch64/distribution/leap/42.2/repo/oss/suse/aarch64/grub2-arm64-efi-2.02~beta2-87.1.aarch64.rpm | rpm2cpio | cpio -di )
+ - wget http://mirrors.kernel.org/ubuntu/pool/main/m/mpfr4/libmpfr4_3.1.4-1_amd64.deb && sudo dpkg -i libmpfr4_3.1.4-1_amd64.deb && rm libmpfr4_3.1.4-1_amd64.deb
 
 env:
   global:

tools/buildman/func_test.py

@@ -175,6 +175,7 @@ class TestFunctional(unittest.TestCase):
     """
     def setUp(self):
         self._base_dir = tempfile.mkdtemp()
+        self._output_dir = tempfile.mkdtemp()
         self._git_dir = os.path.join(self._base_dir, 'src')
         self._buildman_pathname = sys.argv[0]
         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
@@ -207,6 +208,7 @@ class TestFunctional(unittest.TestCase):
 
     def tearDown(self):
         shutil.rmtree(self._base_dir)
+        shutil.rmtree(self._output_dir)
 
     def setupToolchains(self):
         self._toolchains = toolchain.Toolchains()
@@ -421,7 +423,7 @@ class TestFunctional(unittest.TestCase):
     def testCurrentSource(self):
         """Very simple test to invoke buildman on the current source"""
         self.setupToolchains();
-        self._RunControl()
+        self._RunControl('-o', self._output_dir)
         lines = terminal.GetPrintTestLines()
         self.assertIn('Building current source for %d boards' % len(boards),
                       lines[0].text)
@@ -434,7 +436,7 @@ class TestFunctional(unittest.TestCase):
     def testBadToolchain(self):
         """Test that missing toolchains are detected"""
         self.setupToolchains();
-        ret_code = self._RunControl('-b', TEST_BRANCH)
+        ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
         lines = terminal.GetPrintTestLines()
 
         # Buildman always builds the upstream commit as well
@@ -458,13 +460,13 @@ class TestFunctional(unittest.TestCase):
 
     def testBranch(self):
         """Test building a branch with all toolchains present"""
-        self._RunControl('-b', TEST_BRANCH)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
         self.assertEqual(self._builder.count, self._total_builds)
         self.assertEqual(self._builder.fail, 0)
 
     def testCount(self):
         """Test building a specific number of commitst"""
-        self._RunControl('-b', TEST_BRANCH, '-c2')
+        self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir)
         self.assertEqual(self._builder.count, 2 * len(boards))
         self.assertEqual(self._builder.fail, 0)
         # Each board has a mrproper, config, and then one make per commit
@@ -472,34 +474,34 @@ class TestFunctional(unittest.TestCase):
 
     def testIncremental(self):
         """Test building a branch twice - the second time should do nothing"""
-        self._RunControl('-b', TEST_BRANCH)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 
         # Each board has a mrproper, config, and then one make per commit
         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
         self._make_calls = 0
-        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
         self.assertEqual(self._make_calls, 0)
         self.assertEqual(self._builder.count, self._total_builds)
         self.assertEqual(self._builder.fail, 0)
 
     def testForceBuild(self):
         """The -f flag should force a rebuild"""
-        self._RunControl('-b', TEST_BRANCH)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
         self._make_calls = 0
-        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
+        self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False)
         # Each board has a mrproper, config, and then one make per commit
         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 
     def testForceReconfigure(self):
         """The -f flag should force a rebuild"""
-        self._RunControl('-b', TEST_BRANCH, '-C')
+        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
         # Each commit has a mrproper, config and make
         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 
     def testErrors(self):
         """Test handling of build errors"""
         self._error['board2', 1] = 'fred\n'
-        self._RunControl('-b', TEST_BRANCH)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
         self.assertEqual(self._builder.count, self._total_builds)
         self.assertEqual(self._builder.fail, 1)
 
@@ -507,13 +509,13 @@ class TestFunctional(unittest.TestCase):
         # not be rebuilt
         del self._error['board2', 1]
         self._make_calls = 0
-        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
         self.assertEqual(self._builder.count, self._total_builds)
         self.assertEqual(self._make_calls, 0)
         self.assertEqual(self._builder.fail, 1)
 
         # Now use the -F flag to force rebuild of the bad commit
-        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
+        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False)
         self.assertEqual(self._builder.count, self._total_builds)
         self.assertEqual(self._builder.fail, 0)
         self.assertEqual(self._make_calls, 3)

tools/genboardscfg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
@@ -91,7 +91,7 @@ def output_is_new(output):
 
     # Detect a board that has been removed since the current board database
     # was generated
-    with open(output) as f:
+    with open(output, encoding="utf-8") as f:
         for line in f:
             if line[0] == '#' or line == '\n':
                 continue
@@ -118,12 +118,12 @@ class KconfigScanner:
     }
 
     def __init__(self):
-        """Scan all the Kconfig files and create a Config object."""
+        """Scan all the Kconfig files and create a Kconfig object."""
         # Define environment variables referenced from Kconfig
         os.environ['srctree'] = os.getcwd()
         os.environ['UBOOTVERSION'] = 'dummy'
         os.environ['KCONFIG_OBJDIR'] = ''
-        self._conf = kconfiglib.Config(print_warnings=False)
+        self._conf = kconfiglib.Kconfig(warn=False)
 
     def __del__(self):
         """Delete a leftover temporary file before exit.
@@ -165,11 +165,7 @@ class KconfigScanner:
                 else:
                     f.write(line[colon + 1:])
 
-        warnings = self._conf.load_config(self._tmpfile)
+        self._conf.load_config(self._tmpfile)
-        if warnings:
-            for warning in warnings:
-                print '%s: %s' % (defconfig, warning)
-
         try_remove(self._tmpfile)
         self._tmpfile = None
 
@@ -177,8 +173,8 @@ class KconfigScanner:
 
         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
         # Set '-' if the value is empty.
-        for key, symbol in self._SYMBOL_TABLE.items():
+        for key, symbol in list(self._SYMBOL_TABLE.items()):
-            value = self._conf.get_symbol(symbol).get_value()
+            value = self._conf.syms.get(symbol).str_value
             if value:
                 params[key] = value
             else:
@@ -242,8 +238,8 @@ def scan_defconfigs(jobs=1):
     processes = []
     queues = []
     for i in range(jobs):
-        defconfigs = all_defconfigs[total_boards * i / jobs :
+        defconfigs = all_defconfigs[total_boards * i // jobs :
-                                    total_boards * (i + 1) / jobs]
+                                    total_boards * (i + 1) // jobs]
         q = multiprocessing.Queue(maxsize=-1)
         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
                                     args=(q, defconfigs))
@@ -290,7 +286,7 @@ class MaintainersDatabase:
           'Active', 'Orphan' or '-'.
         """
         if not target in self.database:
-            print >> sys.stderr, "WARNING: no status info for '%s'" % target
+            print("WARNING: no status info for '%s'" % target, file=sys.stderr)
             return '-'
 
         tmp = self.database[target][0]
@@ -301,8 +297,8 @@ class MaintainersDatabase:
         elif tmp.startswith('Orphan'):
             return 'Orphan'
         else:
-            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
+            print(("WARNING: %s: unknown status for '%s'" %
-                                  (tmp, target))
+                                  (tmp, target)), file=sys.stderr)
             return '-'
 
     def get_maintainers(self, target):
@@ -313,7 +309,7 @@ class MaintainersDatabase:
           they are separated with colons.
         """
         if not target in self.database:
-            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
+            print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
             return ''
 
         return ':'.join(self.database[target][1])
@@ -330,7 +326,7 @@ class MaintainersDatabase:
         targets = []
         maintainers = []
         status = '-'
-        for line in open(file):
+        for line in open(file, encoding="utf-8"):
             # Check also commented maintainers
             if line[:3] == '#M:':
                 line = line[1:]
@@ -404,7 +400,7 @@ def format_and_output(params_list, output):
     # ignore case when sorting
     output_lines.sort(key=str.lower)
 
-    with open(output, 'w') as f:
+    with open(output, 'w', encoding="utf-8") as f:
         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 
 def gen_boards_cfg(output, jobs=1, force=False):
@@ -418,7 +414,7 @@ def gen_boards_cfg(output, jobs=1, force=False):
     check_top_directory()
 
     if not force and output_is_new(output):
-        print "%s is up to date. Nothing to do." % output
+        print("%s is up to date. Nothing to do." % output)
         sys.exit(0)
 
     params_list = scan_defconfigs(jobs)

tools/moveconfig.py

@@ -851,7 +851,7 @@ class KconfigScanner:
         os.environ['srctree'] = os.getcwd()
         os.environ['UBOOTVERSION'] = 'dummy'
         os.environ['KCONFIG_OBJDIR'] = ''
-        self.conf = kconfiglib.Config()
+        self.conf = kconfiglib.Kconfig()
 
 
 class KconfigParser:
@@ -1525,7 +1525,7 @@ def find_kconfig_rules(kconf, config, imply_config):
     """Check whether a config has a 'select' or 'imply' keyword
 
     Args:
-        kconf: Kconfig.Config object
+        kconf: Kconfiglib.Kconfig object
         config: Name of config to check (without CONFIG_ prefix)
         imply_config: Implying config (without CONFIG_ prefix) which may or
             may not have an 'imply' for 'config')
@@ -1533,7 +1533,7 @@ def find_kconfig_rules(kconf, config, imply_config):
     Returns:
         Symbol object for 'config' if found, else None
     """
-    sym = kconf.get_symbol(imply_config)
+    sym = kconf.syms.get(imply_config)
     if sym:
         for sel in sym.get_selected_symbols() | sym.get_implied_symbols():
             if sel.get_name() == config:
@@ -1547,7 +1547,7 @@ def check_imply_rule(kconf, config, imply_config):
     to add an 'imply' for 'config' to that part of the Kconfig.
 
     Args:
-        kconf: Kconfig.Config object
+        kconf: Kconfiglib.Kconfig object
         config: Name of config to check (without CONFIG_ prefix)
         imply_config: Implying config (without CONFIG_ prefix) which may or
             may not have an 'imply' for 'config')
@@ -1558,7 +1558,7 @@ def check_imply_rule(kconf, config, imply_config):
             line number within the Kconfig file, or 0 if none
             message indicating the result
     """
-    sym = kconf.get_symbol(imply_config)
+    sym = kconf.syms.get(imply_config)
     if not sym:
         return 'cannot find sym'
     locs = sym.get_def_locations()
@@ -1784,7 +1784,7 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added,
                         if skip_added:
                             show = False
                 else:
-                    sym = kconf.get_symbol(iconfig[CONFIG_LEN:])
+                    sym = kconf.syms.get(iconfig[CONFIG_LEN:])
                     fname = ''
                     if sym:
                         locs = sym.get_def_locations()