brain-hackers_pepepper-mirror/u-boot-brain
1
0
Fork 0
mirror of https://github.com/brain-hackers/u-boot-brain synced 2024-10-04 02:20:43 +09:00
Code Issues Projects Releases Wiki Activity

buildman: Allow building of current source tree

Browse Source 
Originally buildman had some support for building the current source tree.
However this was dropped before it was submitted, as part of the effort to
make it faster when building entire branches.

Reinstate this support. If no -b option is given, buildman will build the
current source tree.

Reported-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
Signed-off-by: Simon Glass <sjg@chromium.org>
This commit is contained in:
Simon Glass 2014-08-09 15:32:59 -06:00
parent 6eede34ce6
commit fea5858eb9
2 changed files with 108 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
tools/buildman/builder.py
View File

@ -248,21 +248,34 @@ class BuilderThread(threading.Thread):
if self.toolchain: if self.toolchain:
# Checkout the right commit # Checkout the right commit
if commit_upto is not None: if self.builder.commits:
commit = self.builder.commits[commit_upto] commit = self.builder.commits[commit_upto]
if self.builder.checkout: if self.builder.checkout:
git_dir = os.path.join(work_dir, '.git') git_dir = os.path.join(work_dir, '.git')
gitutil.Checkout(commit.hash, git_dir, work_dir, gitutil.Checkout(commit.hash, git_dir, work_dir,
force=True) force=True)
else: else:
commit = self.builder.commit # Ick, fix this for BuildCommits() commit = 'current'
# Set up the environment and command line # Set up the environment and command line
env = self.toolchain.MakeEnvironment() env = self.toolchain.MakeEnvironment()
Mkdir(out_dir) Mkdir(out_dir)
args = [] args = []
cwd = work_dir
if not self.builder.in_tree: if not self.builder.in_tree:
args.append('O=build') if commit_upto is None:
# In this case we are building in the original source
# directory (i.e. the current directory where buildman
# is invoked. The output directory is set to this
# thread's selected work directory.
#
# Symlinks can confuse U-Boot's Makefile since
# we may use '..' in our path, so remove them.
work_dir = os.path.realpath(work_dir)
args.append('O=%s/build' % work_dir)
cwd = None
else:
args.append('O=build')
args.append('-s') args.append('-s')
if self.builder.num_jobs is not None: if self.builder.num_jobs is not None:
args.extend(['-j', str(self.builder.num_jobs)]) args.extend(['-j', str(self.builder.num_jobs)])
@ -272,14 +285,14 @@ class BuilderThread(threading.Thread):
# If we need to reconfigure, do that now # If we need to reconfigure, do that now
if do_config: if do_config:
result = self.Make(commit, brd, 'distclean', work_dir, result = self.Make(commit, brd, 'distclean', cwd,
'distclean', *args, env=env) 'distclean', *args, env=env)
result = self.Make(commit, brd, 'config', work_dir, result = self.Make(commit, brd, 'config', cwd,
*(args + config_args), env=env) *(args + config_args), env=env)
config_out = result.combined config_out = result.combined
do_config = False # No need to configure next time do_config = False # No need to configure next time
if result.return_code == 0: if result.return_code == 0:
result = self.Make(commit, brd, 'build', work_dir, *args, result = self.Make(commit, brd, 'build', cwd, *args,
env=env) env=env)
result.stdout = config_out + result.stdout result.stdout = config_out + result.stdout
else: else:
@ -478,8 +491,10 @@ class BuilderThread(threading.Thread):
self.builder.out_queue.put(result) self.builder.out_queue.put(result)
else: else:
# Just build the currently checked-out build # Just build the currently checked-out build
result = self.RunCommit(None, True) result, request_config = self.RunCommit(None, brd, work_dir, True,
result.commit_upto = self.builder.upto True, self.builder.force_build_failures)
result.commit_upto = 0
self._WriteResult(result, job.keep_outputs)
self.builder.out_queue.put(result) self.builder.out_queue.put(result)
def run(self): def run(self):
@ -491,12 +506,16 @@ class BuilderThread(threading.Thread):
alive = True alive = True
while True: while True:
job = self.builder.queue.get() job = self.builder.queue.get()
if self.builder.active and alive:
self.RunJob(job)
'''
try: try:
if self.builder.active and alive: if self.builder.active and alive:
self.RunJob(job) self.RunJob(job)
except Exception as err: except Exception as err:
alive = False alive = False
print err print err
'''
self.builder.queue.task_done() self.builder.queue.task_done()
@ -763,10 +782,13 @@ class Builder:
Args: Args:
commit_upto: Commit number to use (0..self.count-1) commit_upto: Commit number to use (0..self.count-1)
""" """
commit = self.commits[commit_upto] if self.commits:
subject = commit.subject.translate(trans_valid_chars) commit = self.commits[commit_upto]
commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, subject = commit.subject.translate(trans_valid_chars)
self.commit_count, commit.hash, subject[:20])) commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
self.commit_count, commit.hash, subject[:20]))
else:
commit_dir = 'current'
output_dir = os.path.join(self.base_dir, commit_dir) output_dir = os.path.join(self.base_dir, commit_dir)
return output_dir return output_dir
@ -1308,14 +1330,18 @@ class Builder:
show_detail: Show detail for each board show_detail: Show detail for each board
show_bloat: Show detail for each function show_bloat: Show detail for each function
""" """
self.commit_count = len(commits) self.commit_count = len(commits) if commits else 1
self.commits = commits self.commits = commits
self.ResetResultSummary(board_selected) self.ResetResultSummary(board_selected)
for commit_upto in range(0, self.commit_count, self._step): for commit_upto in range(0, self.commit_count, self._step):
board_dict, err_lines = self.GetResultSummary(board_selected, board_dict, err_lines = self.GetResultSummary(board_selected,
commit_upto, read_func_sizes=show_bloat) commit_upto, read_func_sizes=show_bloat)
msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) if commits:
msg = '%02d: %s' % (commit_upto + 1,
commits[commit_upto].subject)
else:
msg = 'current'
print self.col.Color(self.col.BLUE, msg) print self.col.Color(self.col.BLUE, msg)
self.PrintResultSummary(board_selected, board_dict, self.PrintResultSummary(board_selected, board_dict,
err_lines if show_errors else [], show_sizes, show_detail, err_lines if show_errors else [], show_sizes, show_detail,
@ -1330,7 +1356,7 @@ class Builder:
commits: Selected commits to build commits: Selected commits to build
""" """
# First work out how many commits we will build # First work out how many commits we will build
count = (len(commits) + self._step - 1) / self._step count = (self.commit_count + self._step - 1) / self._step
self.count = len(board_selected) * count self.count = len(board_selected) * count
self.upto = self.warned = self.fail = 0 self.upto = self.warned = self.fail = 0
self._timestamps = collections.deque() self._timestamps = collections.deque()
@ -1377,13 +1403,14 @@ class Builder:
""" """
return os.path.join(self._working_dir, '%02d' % thread_num) return os.path.join(self._working_dir, '%02d' % thread_num)
def _PrepareThread(self, thread_num): def _PrepareThread(self, thread_num, setup_git):
"""Prepare the working directory for a thread. """Prepare the working directory for a thread.
This clones or fetches the repo into the thread's work directory. This clones or fetches the repo into the thread's work directory.
Args: Args:
thread_num: Thread number (0, 1, ...) thread_num: Thread number (0, 1, ...)
setup_git: True to set up a git repo clone
""" """
thread_dir = self.GetThreadDir(thread_num) thread_dir = self.GetThreadDir(thread_num)
Mkdir(thread_dir) Mkdir(thread_dir)
@ -1392,7 +1419,7 @@ class Builder:
# Clone the repo if it doesn't already exist # Clone the repo if it doesn't already exist
# TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
# we have a private index but uses the origin repo's contents? # we have a private index but uses the origin repo's contents?
if self.git_dir: if setup_git and self.git_dir:
src_dir = os.path.abspath(self.git_dir) src_dir = os.path.abspath(self.git_dir)
if os.path.exists(git_dir): if os.path.exists(git_dir):
gitutil.Fetch(git_dir, thread_dir) gitutil.Fetch(git_dir, thread_dir)
@ -1400,17 +1427,18 @@ class Builder:
print 'Cloning repo for thread %d' % thread_num print 'Cloning repo for thread %d' % thread_num
gitutil.Clone(src_dir, thread_dir) gitutil.Clone(src_dir, thread_dir)
def _PrepareWorkingSpace(self, max_threads): def _PrepareWorkingSpace(self, max_threads, setup_git):
"""Prepare the working directory for use. """Prepare the working directory for use.
Set up the git repo for each thread. Set up the git repo for each thread.
Args: Args:
max_threads: Maximum number of threads we expect to need. max_threads: Maximum number of threads we expect to need.
setup_git: True to set up a git repo clone
""" """
Mkdir(self._working_dir) Mkdir(self._working_dir)
for thread in range(max_threads): for thread in range(max_threads):
self._PrepareThread(thread) self._PrepareThread(thread, setup_git)
def _PrepareOutputSpace(self): def _PrepareOutputSpace(self):
"""Get the output directories ready to receive files. """Get the output directories ready to receive files.
@ -1437,12 +1465,13 @@ class Builder:
show_errors: True to show summarised error/warning info show_errors: True to show summarised error/warning info
keep_outputs: True to save build output files keep_outputs: True to save build output files
""" """
self.commit_count = len(commits) self.commit_count = len(commits) if commits else 1
self.commits = commits self.commits = commits
self.ResetResultSummary(board_selected) self.ResetResultSummary(board_selected)
Mkdir(self.base_dir) Mkdir(self.base_dir)
self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
commits is not None)
self._PrepareOutputSpace() self._PrepareOutputSpace()
self.SetupBuild(board_selected, commits) self.SetupBuild(board_selected, commits)
self.ProcessResult(None) self.ProcessResult(None)

91
tools/buildman/control.py
View File

@ -21,15 +21,20 @@ def GetPlural(count):
"""Returns a plural 's' if count is not 1""" """Returns a plural 's' if count is not 1"""
return 's' if count != 1 else '' return 's' if count != 1 else ''
def GetActionSummary(is_summary, count, selected, options): def GetActionSummary(is_summary, commits, selected, options):
"""Return a string summarising the intended action. """Return a string summarising the intended action.
Returns: Returns:
Summary string. Summary string.
""" """
count = (count + options.step - 1) / options.step if commits:
str = '%s %d commit%s for %d boards' % ( count = len(commits)
'Summary of' if is_summary else 'Building', count, GetPlural(count), count = (count + options.step - 1) / options.step
commit_str = '%d commit%s' % (count, GetPlural(count))
else:
commit_str = 'current source'
str = '%s %s for %d boards' % (
'Summary of' if is_summary else 'Building', commit_str,
len(selected)) len(selected))
str += ' (%d thread%s, %d job%s per thread)' % (options.threads, str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
@ -53,13 +58,18 @@ def ShowActions(series, why_selected, boards_selected, builder, options):
col = terminal.Color() col = terminal.Color()
print 'Dry run, so not doing much. But I would do this:' print 'Dry run, so not doing much. But I would do this:'
print print
print GetActionSummary(False, len(series.commits), boards_selected, if series:
commits = series.commits
else:
commits = None
print GetActionSummary(False, commits, boards_selected,
options) options)
print 'Build directory: %s' % builder.base_dir print 'Build directory: %s' % builder.base_dir
for upto in range(0, len(series.commits), options.step): if commits:
commit = series.commits[upto] for upto in range(0, len(series.commits), options.step):
print ' ', col.Color(col.YELLOW, commit.hash, bright=False), commit = series.commits[upto]
print commit.subject print ' ', col.Color(col.YELLOW, commit.hash, bright=False),
print commit.subject
print print
for arg in why_selected: for arg in why_selected:
if arg != 'all': if arg != 'all':
@ -93,15 +103,16 @@ def DoBuildman(options, args):
count = options.count count = options.count
if count == -1: if count == -1:
if not options.branch: if not options.branch:
str = 'Please use -b to specify a branch to build' count = 1
print col.Color(col.RED, str) else:
sys.exit(1) count = gitutil.CountCommitsInBranch(options.git_dir,
count = gitutil.CountCommitsInBranch(options.git_dir, options.branch) options.branch)
if count is None: if count is None:
str = "Branch '%s' not found or has no upstream" % options.branch str = ("Branch '%s' not found or has no upstream" %
print col.Color(col.RED, str) options.branch)
sys.exit(1) print col.Color(col.RED, str)
count += 1 # Build upstream commit also sys.exit(1)
count += 1 # Build upstream commit also
if not count: if not count:
str = ("No commits found to process in branch '%s': " str = ("No commits found to process in branch '%s': "
@ -132,17 +143,21 @@ def DoBuildman(options, args):
# upstream/master~..branch but that isn't possible if upstream/master is # upstream/master~..branch but that isn't possible if upstream/master is
# a merge commit (it will list all the commits that form part of the # a merge commit (it will list all the commits that form part of the
# merge) # merge)
range_expr = gitutil.GetRangeInBranch(options.git_dir, options.branch) if options.branch:
upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch) range_expr = gitutil.GetRangeInBranch(options.git_dir, options.branch)
series = patchstream.GetMetaDataForList(upstream_commit, options.git_dir, upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch)
1) series = patchstream.GetMetaDataForList(upstream_commit,
# Conflicting tags are not a problem for buildman, since it does not use options.git_dir, 1)
# them. For example, Series-version is not useful for buildman. On the
# other hand conflicting tags will cause an error. So allow later tags # Conflicting tags are not a problem for buildman, since it does not
# to overwrite earlier ones. # use them. For example, Series-version is not useful for buildman. On
series.allow_overwrite = True # the other hand conflicting tags will cause an error. So allow later
series = patchstream.GetMetaDataForList(range_expr, options.git_dir, None, # tags to overwrite earlier ones.
series) series.allow_overwrite = True
series = patchstream.GetMetaDataForList(range_expr, options.git_dir, None,
series)
else:
series = None
# By default we have one thread per CPU. But if there are not enough jobs # By default we have one thread per CPU. But if there are not enough jobs
# we can have fewer threads and use a high '-j' value for make. # we can have fewer threads and use a high '-j' value for make.
@ -162,7 +177,11 @@ def DoBuildman(options, args):
sys.exit(1) sys.exit(1)
# Create a new builder with the selected options # Create a new builder with the selected options
output_dir = os.path.join(options.output_dir, options.branch) if options.branch:
dirname = options.branch
else:
dirname = 'current'
output_dir = os.path.join(options.output_dir, dirname)
builder = Builder(toolchains, output_dir, options.git_dir, builder = Builder(toolchains, output_dir, options.git_dir,
options.threads, options.jobs, gnu_make=gnu_make, checkout=True, options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
show_unknown=options.show_unknown, step=options.step) show_unknown=options.show_unknown, step=options.step)
@ -180,15 +199,21 @@ def DoBuildman(options, args):
# Work out which boards to build # Work out which boards to build
board_selected = boards.GetSelectedDict() board_selected = boards.GetSelectedDict()
print GetActionSummary(options.summary, count, board_selected, options) if series:
commits = series.commits
else:
commits = None
print GetActionSummary(options.summary, commits, board_selected,
options)
if options.summary: if options.summary:
# We can't show function sizes without board details at present # We can't show function sizes without board details at present
if options.show_bloat: if options.show_bloat:
options.show_detail = True options.show_detail = True
builder.ShowSummary(series.commits, board_selected, builder.ShowSummary(commits, board_selected,
options.show_errors, options.show_sizes, options.show_errors, options.show_sizes,
options.show_detail, options.show_bloat) options.show_detail, options.show_bloat)
else: else:
builder.BuildBoards(series.commits, board_selected, builder.BuildBoards(commits, board_selected,
options.show_errors, options.keep_outputs) options.show_errors, options.keep_outputs)