mirror of
https://github.com/brain-hackers/u-boot-brain
synced 2024-06-09 23:36:03 +09:00
buildman: Support single-threaded operation
At present even if only a single thread is in use, buildman still uses threading. For some debugging it is helpful to do everything in the main process. Allow -T0 to support this. Signed-off-by: Simon Glass <sjg@chromium.org>
This commit is contained in:
parent
d6bf36c775
commit
b82492bbcc
|
@ -1128,6 +1128,11 @@ If there are both warnings and errors, errors win, so buildman returns 100.
|
||||||
The -y option is provided (for use with -s) to ignore the bountiful device-tree
|
The -y option is provided (for use with -s) to ignore the bountiful device-tree
|
||||||
warnings. Similarly, -Y tells buildman to ignore the migration warnings.
|
warnings. Similarly, -Y tells buildman to ignore the migration warnings.
|
||||||
|
|
||||||
|
Sometimes you might get an error in a thread that is not handled by buildman,
|
||||||
|
perhaps due to a failure of a tool that it calls. You might see the output, but
|
||||||
|
then buildman hangs. Failing to handle any eventuality is a bug in buildman and
|
||||||
|
should be reported. But you can use -T0 to disable threading and hopefully
|
||||||
|
figure out the root cause of the build failure.
|
||||||
|
|
||||||
Build summary
|
Build summary
|
||||||
=============
|
=============
|
||||||
|
|
|
@ -197,6 +197,8 @@ class Builder:
|
||||||
last _timestamp_count builds. Each is a datetime object.
|
last _timestamp_count builds. Each is a datetime object.
|
||||||
_timestamp_count: Number of timestamps to keep in our list.
|
_timestamp_count: Number of timestamps to keep in our list.
|
||||||
_working_dir: Base working directory containing all threads
|
_working_dir: Base working directory containing all threads
|
||||||
|
_single_builder: BuilderThread object for the singer builder, if
|
||||||
|
threading is not being used
|
||||||
"""
|
"""
|
||||||
class Outcome:
|
class Outcome:
|
||||||
"""Records a build outcome for a single make invocation
|
"""Records a build outcome for a single make invocation
|
||||||
|
@ -309,19 +311,24 @@ class Builder:
|
||||||
self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
|
self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
|
||||||
re.MULTILINE | re.DOTALL)
|
re.MULTILINE | re.DOTALL)
|
||||||
|
|
||||||
self.queue = queue.Queue()
|
if self.num_threads:
|
||||||
self.out_queue = queue.Queue()
|
self._single_builder = None
|
||||||
for i in range(self.num_threads):
|
self.queue = queue.Queue()
|
||||||
t = builderthread.BuilderThread(self, i, mrproper,
|
self.out_queue = queue.Queue()
|
||||||
per_board_out_dir)
|
for i in range(self.num_threads):
|
||||||
|
t = builderthread.BuilderThread(self, i, mrproper,
|
||||||
|
per_board_out_dir)
|
||||||
|
t.setDaemon(True)
|
||||||
|
t.start()
|
||||||
|
self.threads.append(t)
|
||||||
|
|
||||||
|
t = builderthread.ResultThread(self)
|
||||||
t.setDaemon(True)
|
t.setDaemon(True)
|
||||||
t.start()
|
t.start()
|
||||||
self.threads.append(t)
|
self.threads.append(t)
|
||||||
|
else:
|
||||||
t = builderthread.ResultThread(self)
|
self._single_builder = builderthread.BuilderThread(
|
||||||
t.setDaemon(True)
|
self, -1, mrproper, per_board_out_dir)
|
||||||
t.start()
|
|
||||||
self.threads.append(t)
|
|
||||||
|
|
||||||
ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
|
ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
|
||||||
self.re_make_err = re.compile('|'.join(ignore_lines))
|
self.re_make_err = re.compile('|'.join(ignore_lines))
|
||||||
|
@ -1531,11 +1538,12 @@ class Builder:
|
||||||
"""Get the directory path to the working dir for a thread.
|
"""Get the directory path to the working dir for a thread.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
thread_num: Number of thread to check.
|
thread_num: Number of thread to check (-1 for main process, which
|
||||||
|
is treated as 0)
|
||||||
"""
|
"""
|
||||||
if self.work_in_output:
|
if self.work_in_output:
|
||||||
return self._working_dir
|
return self._working_dir
|
||||||
return os.path.join(self._working_dir, '%02d' % thread_num)
|
return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
|
||||||
|
|
||||||
def _PrepareThread(self, thread_num, setup_git):
|
def _PrepareThread(self, thread_num, setup_git):
|
||||||
"""Prepare the working directory for a thread.
|
"""Prepare the working directory for a thread.
|
||||||
|
@ -1594,7 +1602,9 @@ class Builder:
|
||||||
if git-worktree is available, or clones the repo if it isn't.
|
if git-worktree is available, or clones the repo if it isn't.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
max_threads: Maximum number of threads we expect to need.
|
max_threads: Maximum number of threads we expect to need. If 0 then
|
||||||
|
1 is set up, since the main process still needs somewhere to
|
||||||
|
work
|
||||||
setup_git: True to set up a git worktree or a git clone
|
setup_git: True to set up a git worktree or a git clone
|
||||||
"""
|
"""
|
||||||
builderthread.Mkdir(self._working_dir)
|
builderthread.Mkdir(self._working_dir)
|
||||||
|
@ -1608,7 +1618,9 @@ class Builder:
|
||||||
gitutil.PruneWorktrees(src_dir)
|
gitutil.PruneWorktrees(src_dir)
|
||||||
else:
|
else:
|
||||||
setup_git = 'clone'
|
setup_git = 'clone'
|
||||||
for thread in range(max_threads):
|
|
||||||
|
# Always do at least one thread
|
||||||
|
for thread in range(max(max_threads, 1)):
|
||||||
self._PrepareThread(thread, setup_git)
|
self._PrepareThread(thread, setup_git)
|
||||||
|
|
||||||
def _GetOutputSpaceRemovals(self):
|
def _GetOutputSpaceRemovals(self):
|
||||||
|
@ -1686,16 +1698,20 @@ class Builder:
|
||||||
job.keep_outputs = keep_outputs
|
job.keep_outputs = keep_outputs
|
||||||
job.work_in_output = self.work_in_output
|
job.work_in_output = self.work_in_output
|
||||||
job.step = self._step
|
job.step = self._step
|
||||||
self.queue.put(job)
|
if self.num_threads:
|
||||||
|
self.queue.put(job)
|
||||||
|
else:
|
||||||
|
results = self._single_builder.RunJob(job)
|
||||||
|
|
||||||
term = threading.Thread(target=self.queue.join)
|
if self.num_threads:
|
||||||
term.setDaemon(True)
|
term = threading.Thread(target=self.queue.join)
|
||||||
term.start()
|
term.setDaemon(True)
|
||||||
while term.is_alive():
|
term.start()
|
||||||
term.join(100)
|
while term.is_alive():
|
||||||
|
term.join(100)
|
||||||
|
|
||||||
# Wait until we have processed all output
|
# Wait until we have processed all output
|
||||||
self.out_queue.join()
|
self.out_queue.join()
|
||||||
Print()
|
Print()
|
||||||
|
|
||||||
msg = 'Completed: %d total built' % self.count
|
msg = 'Completed: %d total built' % self.count
|
||||||
|
|
|
@ -89,7 +89,8 @@ class BuilderThread(threading.Thread):
|
||||||
Members:
|
Members:
|
||||||
builder: The builder which contains information we might need
|
builder: The builder which contains information we might need
|
||||||
thread_num: Our thread number (0-n-1), used to decide on a
|
thread_num: Our thread number (0-n-1), used to decide on a
|
||||||
temporary directory
|
temporary directory. If this is -1 then there are no threads
|
||||||
|
and we are the (only) main process
|
||||||
"""
|
"""
|
||||||
def __init__(self, builder, thread_num, mrproper, per_board_out_dir):
|
def __init__(self, builder, thread_num, mrproper, per_board_out_dir):
|
||||||
"""Set up a new builder thread"""
|
"""Set up a new builder thread"""
|
||||||
|
@ -445,6 +446,9 @@ class BuilderThread(threading.Thread):
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
job: Job to build
|
job: Job to build
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of Result objects
|
||||||
"""
|
"""
|
||||||
brd = job.board
|
brd = job.board
|
||||||
work_dir = self.builder.GetThreadDir(self.thread_num)
|
work_dir = self.builder.GetThreadDir(self.thread_num)
|
||||||
|
@ -508,7 +512,10 @@ class BuilderThread(threading.Thread):
|
||||||
|
|
||||||
# We have the build results, so output the result
|
# We have the build results, so output the result
|
||||||
self._WriteResult(result, job.keep_outputs, job.work_in_output)
|
self._WriteResult(result, job.keep_outputs, job.work_in_output)
|
||||||
self.builder.out_queue.put(result)
|
if self.thread_num != -1:
|
||||||
|
self.builder.out_queue.put(result)
|
||||||
|
else:
|
||||||
|
self.builder.ProcessResult(result)
|
||||||
else:
|
else:
|
||||||
# Just build the currently checked-out build
|
# Just build the currently checked-out build
|
||||||
result, request_config = self.RunCommit(None, brd, work_dir, True,
|
result, request_config = self.RunCommit(None, brd, work_dir, True,
|
||||||
|
@ -517,7 +524,10 @@ class BuilderThread(threading.Thread):
|
||||||
work_in_output=job.work_in_output)
|
work_in_output=job.work_in_output)
|
||||||
result.commit_upto = 0
|
result.commit_upto = 0
|
||||||
self._WriteResult(result, job.keep_outputs, job.work_in_output)
|
self._WriteResult(result, job.keep_outputs, job.work_in_output)
|
||||||
self.builder.out_queue.put(result)
|
if self.thread_num != -1:
|
||||||
|
self.builder.out_queue.put(result)
|
||||||
|
else:
|
||||||
|
self.builder.ProcessResult(result)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""Our thread's run function
|
"""Our thread's run function
|
||||||
|
|
|
@ -97,7 +97,8 @@ def ParseArgs():
|
||||||
parser.add_option('-t', '--test', action='store_true', dest='test',
|
parser.add_option('-t', '--test', action='store_true', dest='test',
|
||||||
default=False, help='run tests')
|
default=False, help='run tests')
|
||||||
parser.add_option('-T', '--threads', type='int',
|
parser.add_option('-T', '--threads', type='int',
|
||||||
default=None, help='Number of builder threads to use')
|
default=None,
|
||||||
|
help='Number of builder threads to use (0=single-thread)')
|
||||||
parser.add_option('-u', '--show_unknown', action='store_true',
|
parser.add_option('-u', '--show_unknown', action='store_true',
|
||||||
default=False, help='Show boards with unknown build result')
|
default=False, help='Show boards with unknown build result')
|
||||||
parser.add_option('-U', '--show-environment', action='store_true',
|
parser.add_option('-U', '--show-environment', action='store_true',
|
||||||
|
|
|
@ -294,7 +294,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
|
||||||
|
|
||||||
# By default we have one thread per CPU. But if there are not enough jobs
|
# By default we have one thread per CPU. But if there are not enough jobs
|
||||||
# we can have fewer threads and use a high '-j' value for make.
|
# we can have fewer threads and use a high '-j' value for make.
|
||||||
if not options.threads:
|
if options.threads is None:
|
||||||
options.threads = min(multiprocessing.cpu_count(), len(selected))
|
options.threads = min(multiprocessing.cpu_count(), len(selected))
|
||||||
if not options.jobs:
|
if not options.jobs:
|
||||||
options.jobs = max(1, (multiprocessing.cpu_count() +
|
options.jobs = max(1, (multiprocessing.cpu_count() +
|
||||||
|
|
|
@ -187,7 +187,7 @@ class TestBuild(unittest.TestCase):
|
||||||
expect += col.Color(expected_colour, ' %s' % board)
|
expect += col.Color(expected_colour, ' %s' % board)
|
||||||
self.assertEqual(text, expect)
|
self.assertEqual(text, expect)
|
||||||
|
|
||||||
def _SetupTest(self, echo_lines=False, **kwdisplay_args):
|
def _SetupTest(self, echo_lines=False, threads=1, **kwdisplay_args):
|
||||||
"""Set up the test by running a build and summary
|
"""Set up the test by running a build and summary
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
@ -199,8 +199,8 @@ class TestBuild(unittest.TestCase):
|
||||||
Returns:
|
Returns:
|
||||||
Iterator containing the output lines, each a PrintLine() object
|
Iterator containing the output lines, each a PrintLine() object
|
||||||
"""
|
"""
|
||||||
build = builder.Builder(self.toolchains, self.base_dir, None, 1, 2,
|
build = builder.Builder(self.toolchains, self.base_dir, None, threads,
|
||||||
checkout=False, show_unknown=False)
|
2, checkout=False, show_unknown=False)
|
||||||
build.do_make = self.Make
|
build.do_make = self.Make
|
||||||
board_selected = self.boards.GetSelectedDict()
|
board_selected = self.boards.GetSelectedDict()
|
||||||
|
|
||||||
|
@ -438,6 +438,12 @@ class TestBuild(unittest.TestCase):
|
||||||
filter_migration_warnings=True)
|
filter_migration_warnings=True)
|
||||||
self._CheckOutput(lines, filter_migration_warnings=True)
|
self._CheckOutput(lines, filter_migration_warnings=True)
|
||||||
|
|
||||||
|
def testSingleThread(self):
|
||||||
|
"""Test operation without threading"""
|
||||||
|
lines = self._SetupTest(show_errors=True, threads=0)
|
||||||
|
self._CheckOutput(lines, list_error_boards=False,
|
||||||
|
filter_dtb_warnings=False)
|
||||||
|
|
||||||
def _testGit(self):
|
def _testGit(self):
|
||||||
"""Test basic builder operation by building a branch"""
|
"""Test basic builder operation by building a branch"""
|
||||||
options = Options()
|
options = Options()
|
||||||
|
|
Loading…
Reference in New Issue
Block a user