386 lines
15 KiB
Python
386 lines
15 KiB
Python
# SPDX-License-Identifier: GPL-2.0+
|
|
# Copyright (c) 2013 The Chromium OS Authors.
|
|
#
|
|
|
|
import multiprocessing
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
from buildman import board
|
|
from buildman import bsettings
|
|
from buildman import toolchain
|
|
from buildman.builder import Builder
|
|
from patman import command
|
|
from patman import gitutil
|
|
from patman import patchstream
|
|
from patman import terminal
|
|
from patman.terminal import Print
|
|
|
|
def GetPlural(count):
|
|
"""Returns a plural 's' if count is not 1"""
|
|
return 's' if count != 1 else ''
|
|
|
|
def GetActionSummary(is_summary, commits, selected, options):
|
|
"""Return a string summarising the intended action.
|
|
|
|
Returns:
|
|
Summary string.
|
|
"""
|
|
if commits:
|
|
count = len(commits)
|
|
count = (count + options.step - 1) // options.step
|
|
commit_str = '%d commit%s' % (count, GetPlural(count))
|
|
else:
|
|
commit_str = 'current source'
|
|
str = '%s %s for %d boards' % (
|
|
'Summary of' if is_summary else 'Building', commit_str,
|
|
len(selected))
|
|
str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
|
|
GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
|
|
return str
|
|
|
|
def ShowActions(series, why_selected, boards_selected, builder, options,
|
|
board_warnings):
|
|
"""Display a list of actions that we would take, if not a dry run.
|
|
|
|
Args:
|
|
series: Series object
|
|
why_selected: Dictionary where each key is a buildman argument
|
|
provided by the user, and the value is the list of boards
|
|
brought in by that argument. For example, 'arm' might bring
|
|
in 400 boards, so in this case the key would be 'arm' and
|
|
the value would be a list of board names.
|
|
boards_selected: Dict of selected boards, key is target name,
|
|
value is Board object
|
|
builder: The builder that will be used to build the commits
|
|
options: Command line options object
|
|
board_warnings: List of warnings obtained from board selected
|
|
"""
|
|
col = terminal.Color()
|
|
print('Dry run, so not doing much. But I would do this:')
|
|
print()
|
|
if series:
|
|
commits = series.commits
|
|
else:
|
|
commits = None
|
|
print(GetActionSummary(False, commits, boards_selected,
|
|
options))
|
|
print('Build directory: %s' % builder.base_dir)
|
|
if commits:
|
|
for upto in range(0, len(series.commits), options.step):
|
|
commit = series.commits[upto]
|
|
print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
|
|
print(commit.subject)
|
|
print()
|
|
for arg in why_selected:
|
|
if arg != 'all':
|
|
print(arg, ': %d boards' % len(why_selected[arg]))
|
|
if options.verbose:
|
|
print(' %s' % ' '.join(why_selected[arg]))
|
|
print(('Total boards to build for each commit: %d\n' %
|
|
len(why_selected['all'])))
|
|
if board_warnings:
|
|
for warning in board_warnings:
|
|
print(col.Color(col.YELLOW, warning))
|
|
|
|
def ShowToolchainPrefix(boards, toolchains):
|
|
"""Show information about a the tool chain used by one or more boards
|
|
|
|
The function checks that all boards use the same toolchain, then prints
|
|
the correct value for CROSS_COMPILE.
|
|
|
|
Args:
|
|
boards: Boards object containing selected boards
|
|
toolchains: Toolchains object containing available toolchains
|
|
|
|
Return:
|
|
None on success, string error message otherwise
|
|
"""
|
|
boards = boards.GetSelectedDict()
|
|
tc_set = set()
|
|
for brd in boards.values():
|
|
tc_set.add(toolchains.Select(brd.arch))
|
|
if len(tc_set) != 1:
|
|
return 'Supplied boards must share one toolchain'
|
|
return False
|
|
tc = tc_set.pop()
|
|
print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
|
|
return None
|
|
|
|
def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
|
|
clean_dir=False, test_thread_exceptions=False):
|
|
"""The main control code for buildman
|
|
|
|
Args:
|
|
options: Command line options object
|
|
args: Command line arguments (list of strings)
|
|
toolchains: Toolchains to use - this should be a Toolchains()
|
|
object. If None, then it will be created and scanned
|
|
make_func: Make function to use for the builder. This is called
|
|
to execute 'make'. If this is None, the normal function
|
|
will be used, which calls the 'make' tool with suitable
|
|
arguments. This setting is useful for tests.
|
|
board: Boards() object to use, containing a list of available
|
|
boards. If this is None it will be created and scanned.
|
|
clean_dir: Used for tests only, indicates that the existing output_dir
|
|
should be removed before starting the build
|
|
test_thread_exceptions: Uses for tests only, True to make the threads
|
|
raise an exception instead of reporting their result. This simulates
|
|
a failure in the code somewhere
|
|
"""
|
|
global builder
|
|
|
|
if options.full_help:
|
|
pager = os.getenv('PAGER')
|
|
if not pager:
|
|
pager = 'more'
|
|
fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
|
|
'README')
|
|
command.Run(pager, fname)
|
|
return 0
|
|
|
|
gitutil.Setup()
|
|
col = terminal.Color()
|
|
|
|
options.git_dir = os.path.join(options.git, '.git')
|
|
|
|
no_toolchains = toolchains is None
|
|
if no_toolchains:
|
|
toolchains = toolchain.Toolchains(options.override_toolchain)
|
|
|
|
if options.fetch_arch:
|
|
if options.fetch_arch == 'list':
|
|
sorted_list = toolchains.ListArchs()
|
|
print(col.Color(col.BLUE, 'Available architectures: %s\n' %
|
|
' '.join(sorted_list)))
|
|
return 0
|
|
else:
|
|
fetch_arch = options.fetch_arch
|
|
if fetch_arch == 'all':
|
|
fetch_arch = ','.join(toolchains.ListArchs())
|
|
print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
|
|
fetch_arch))
|
|
for arch in fetch_arch.split(','):
|
|
print()
|
|
ret = toolchains.FetchAndInstall(arch)
|
|
if ret:
|
|
return ret
|
|
return 0
|
|
|
|
if no_toolchains:
|
|
toolchains.GetSettings()
|
|
toolchains.Scan(options.list_tool_chains and options.verbose)
|
|
if options.list_tool_chains:
|
|
toolchains.List()
|
|
print()
|
|
return 0
|
|
|
|
if options.incremental:
|
|
print(col.Color(col.RED,
|
|
'Warning: -I has been removed. See documentation'))
|
|
if not options.output_dir:
|
|
if options.work_in_output:
|
|
sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
|
|
options.output_dir = '..'
|
|
|
|
# Work out what subset of the boards we are building
|
|
if not boards:
|
|
if not os.path.exists(options.output_dir):
|
|
os.makedirs(options.output_dir)
|
|
board_file = os.path.join(options.output_dir, 'boards.cfg')
|
|
our_path = os.path.dirname(os.path.realpath(__file__))
|
|
genboardscfg = os.path.join(our_path, '../genboardscfg.py')
|
|
if not os.path.exists(genboardscfg):
|
|
genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
|
|
status = subprocess.call([genboardscfg, '-q', '-o', board_file])
|
|
if status != 0:
|
|
# Older versions don't support -q
|
|
status = subprocess.call([genboardscfg, '-o', board_file])
|
|
if status != 0:
|
|
sys.exit("Failed to generate boards.cfg")
|
|
|
|
boards = board.Boards()
|
|
boards.ReadBoards(board_file)
|
|
|
|
exclude = []
|
|
if options.exclude:
|
|
for arg in options.exclude:
|
|
exclude += arg.split(',')
|
|
|
|
if options.boards:
|
|
requested_boards = []
|
|
for b in options.boards:
|
|
requested_boards += b.split(',')
|
|
else:
|
|
requested_boards = None
|
|
why_selected, board_warnings = boards.SelectBoards(args, exclude,
|
|
requested_boards)
|
|
selected = boards.GetSelected()
|
|
if not len(selected):
|
|
sys.exit(col.Color(col.RED, 'No matching boards found'))
|
|
|
|
if options.print_prefix:
|
|
err = ShowToolchainPrefix(boards, toolchains)
|
|
if err:
|
|
sys.exit(col.Color(col.RED, err))
|
|
return 0
|
|
|
|
# Work out how many commits to build. We want to build everything on the
|
|
# branch. We also build the upstream commit as a control so we can see
|
|
# problems introduced by the first commit on the branch.
|
|
count = options.count
|
|
has_range = options.branch and '..' in options.branch
|
|
if count == -1:
|
|
if not options.branch:
|
|
count = 1
|
|
else:
|
|
if has_range:
|
|
count, msg = gitutil.CountCommitsInRange(options.git_dir,
|
|
options.branch)
|
|
else:
|
|
count, msg = gitutil.CountCommitsInBranch(options.git_dir,
|
|
options.branch)
|
|
if count is None:
|
|
sys.exit(col.Color(col.RED, msg))
|
|
elif count == 0:
|
|
sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
|
|
options.branch))
|
|
if msg:
|
|
print(col.Color(col.YELLOW, msg))
|
|
count += 1 # Build upstream commit also
|
|
|
|
if not count:
|
|
str = ("No commits found to process in branch '%s': "
|
|
"set branch's upstream or use -c flag" % options.branch)
|
|
sys.exit(col.Color(col.RED, str))
|
|
if options.work_in_output:
|
|
if len(selected) != 1:
|
|
sys.exit(col.Color(col.RED,
|
|
'-w can only be used with a single board'))
|
|
if count != 1:
|
|
sys.exit(col.Color(col.RED,
|
|
'-w can only be used with a single commit'))
|
|
|
|
# Read the metadata from the commits. First look at the upstream commit,
|
|
# then the ones in the branch. We would like to do something like
|
|
# upstream/master~..branch but that isn't possible if upstream/master is
|
|
# a merge commit (it will list all the commits that form part of the
|
|
# merge)
|
|
# Conflicting tags are not a problem for buildman, since it does not use
|
|
# them. For example, Series-version is not useful for buildman. On the
|
|
# other hand conflicting tags will cause an error. So allow later tags
|
|
# to overwrite earlier ones by setting allow_overwrite=True
|
|
if options.branch:
|
|
if count == -1:
|
|
if has_range:
|
|
range_expr = options.branch
|
|
else:
|
|
range_expr = gitutil.GetRangeInBranch(options.git_dir,
|
|
options.branch)
|
|
upstream_commit = gitutil.GetUpstream(options.git_dir,
|
|
options.branch)
|
|
series = patchstream.get_metadata_for_list(upstream_commit,
|
|
options.git_dir, 1, series=None, allow_overwrite=True)
|
|
|
|
series = patchstream.get_metadata_for_list(range_expr,
|
|
options.git_dir, None, series, allow_overwrite=True)
|
|
else:
|
|
# Honour the count
|
|
series = patchstream.get_metadata_for_list(options.branch,
|
|
options.git_dir, count, series=None, allow_overwrite=True)
|
|
else:
|
|
series = None
|
|
if not options.dry_run:
|
|
options.verbose = True
|
|
if not options.summary:
|
|
options.show_errors = True
|
|
|
|
# By default we have one thread per CPU. But if there are not enough jobs
|
|
# we can have fewer threads and use a high '-j' value for make.
|
|
if options.threads is None:
|
|
options.threads = min(multiprocessing.cpu_count(), len(selected))
|
|
if not options.jobs:
|
|
options.jobs = max(1, (multiprocessing.cpu_count() +
|
|
len(selected) - 1) // len(selected))
|
|
|
|
if not options.step:
|
|
options.step = len(series.commits) - 1
|
|
|
|
gnu_make = command.Output(os.path.join(options.git,
|
|
'scripts/show-gnu-make'), raise_on_error=False).rstrip()
|
|
if not gnu_make:
|
|
sys.exit('GNU Make not found')
|
|
|
|
# Create a new builder with the selected options.
|
|
output_dir = options.output_dir
|
|
if options.branch:
|
|
dirname = options.branch.replace('/', '_')
|
|
# As a special case allow the board directory to be placed in the
|
|
# output directory itself rather than any subdirectory.
|
|
if not options.no_subdirs:
|
|
output_dir = os.path.join(options.output_dir, dirname)
|
|
if clean_dir and os.path.exists(output_dir):
|
|
shutil.rmtree(output_dir)
|
|
builder = Builder(toolchains, output_dir, options.git_dir,
|
|
options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
|
|
show_unknown=options.show_unknown, step=options.step,
|
|
no_subdirs=options.no_subdirs, full_path=options.full_path,
|
|
verbose_build=options.verbose_build,
|
|
mrproper=options.mrproper,
|
|
per_board_out_dir=options.per_board_out_dir,
|
|
config_only=options.config_only,
|
|
squash_config_y=not options.preserve_config_y,
|
|
warnings_as_errors=options.warnings_as_errors,
|
|
work_in_output=options.work_in_output,
|
|
test_thread_exceptions=test_thread_exceptions)
|
|
builder.force_config_on_failure = not options.quick
|
|
if make_func:
|
|
builder.do_make = make_func
|
|
|
|
# For a dry run, just show our actions as a sanity check
|
|
if options.dry_run:
|
|
ShowActions(series, why_selected, selected, builder, options,
|
|
board_warnings)
|
|
else:
|
|
builder.force_build = options.force_build
|
|
builder.force_build_failures = options.force_build_failures
|
|
builder.force_reconfig = options.force_reconfig
|
|
builder.in_tree = options.in_tree
|
|
|
|
# Work out which boards to build
|
|
board_selected = boards.GetSelectedDict()
|
|
|
|
if series:
|
|
commits = series.commits
|
|
# Number the commits for test purposes
|
|
for commit in range(len(commits)):
|
|
commits[commit].sequence = commit
|
|
else:
|
|
commits = None
|
|
|
|
Print(GetActionSummary(options.summary, commits, board_selected,
|
|
options))
|
|
|
|
# We can't show function sizes without board details at present
|
|
if options.show_bloat:
|
|
options.show_detail = True
|
|
builder.SetDisplayOptions(
|
|
options.show_errors, options.show_sizes, options.show_detail,
|
|
options.show_bloat, options.list_error_boards, options.show_config,
|
|
options.show_environment, options.filter_dtb_warnings,
|
|
options.filter_migration_warnings)
|
|
if options.summary:
|
|
builder.ShowSummary(commits, board_selected)
|
|
else:
|
|
fail, warned, excs = builder.BuildBoards(
|
|
commits, board_selected, options.keep_outputs, options.verbose)
|
|
if excs:
|
|
return 102
|
|
elif fail:
|
|
return 100
|
|
elif warned and not options.ignore_warnings:
|
|
return 101
|
|
return 0
|