From b6bfe1420517aa5aa190e17dca40ea70f09effd9 Mon Sep 17 00:00:00 2001
From: Richard Purdie <rpurdie@linux.intel.com>
Date: Mon, 16 Aug 2010 16:37:29 +0100
Subject: bitbake: Switch to use subprocess for forking tasks and FAKEROOTENV
to run shell and python under a fakeroot environment
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
---
bitbake/lib/bb/build.py | 64 ++++++++++++++--------
bitbake/lib/bb/cooker.py | 7 ++-
bitbake/lib/bb/data.py | 9 +++
bitbake/lib/bb/event.py | 11 ++--
bitbake/lib/bb/msg.py | 11 ++--
bitbake/lib/bb/runqueue.py | 134 ++++++++++++++++++++-------------------------
6 files changed, 124 insertions(+), 112 deletions(-)
(limited to 'bitbake/lib')
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 0bf0154cb..77af92abe 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -44,12 +44,6 @@ class FuncFailed(Exception):
Second paramter is a logfile (optional)
"""
-class EventException(Exception):
- """Exception which is associated with an Event."""
-
- def __init__(self, msg, event):
- self.args = msg, event
-
class TaskBase(event.Event):
"""Base class for task events"""
@@ -80,7 +74,7 @@ class TaskFailed(TaskBase):
self.msg = msg
TaskBase.__init__(self, t, d)
-class InvalidTask(TaskBase):
+class TaskInvalid(TaskBase):
"""Invalid Task"""
# functions
@@ -94,7 +88,7 @@ def exec_func(func, d, dirs = None):
return
flags = data.getVarFlags(func, d)
- for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']:
+ for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot', 'task']:
if not item in flags:
flags[item] = None
@@ -138,7 +132,7 @@ def exec_func(func, d, dirs = None):
# Handle logfiles
si = file('/dev/null', 'r')
try:
- if bb.msg.debug_level['default'] > 0 or ispython:
+ if bb.msg.debug_level['default'] > 0 and not ispython:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
@@ -158,6 +152,8 @@ def exec_func(func, d, dirs = None):
os.dup2(so.fileno(), oso[1])
os.dup2(se.fileno(), ose[1])
+ bb.event.useStdout = True
+
locks = []
lockfiles = flags['lockfiles']
if lockfiles:
@@ -183,6 +179,8 @@ def exec_func(func, d, dirs = None):
for lock in locks:
bb.utils.unlockfile(lock)
+ bb.event.useStdout = False
+
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
@@ -221,6 +219,7 @@ def exec_func_python(func, d, runfile, logfile):
raise
raise FuncFailed("Function %s failed" % func, logfile)
+
def exec_func_shell(func, d, runfile, logfile, flags):
"""Execute a shell BB 'function' Returns true if execution was successful.
@@ -251,12 +250,11 @@ def exec_func_shell(func, d, runfile, logfile, flags):
raise FuncFailed("Function not specified for exec_func_shell")
# execute function
- if flags['fakeroot']:
- maybe_fakeroot = "PATH=\"%s\" %s " % (bb.data.getVar("PATH", d, 1), bb.data.getVar("FAKEROOT", d, 1) or "fakeroot")
- else:
- maybe_fakeroot = ''
+ if flags['fakeroot'] and not flags['task']:
+ bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
+
lang_environment = "LC_ALL=C "
- ret = os.system('%s%ssh -e %s' % (lang_environment, maybe_fakeroot, runfile))
+ ret = os.system('%ssh -e %s' % (lang_environment, runfile))
if ret == 0:
return
@@ -273,7 +271,13 @@ def exec_task(task, d):
# Check whther this is a valid task
if not data.getVarFlag(task, 'task', d):
- raise EventException("No such task", InvalidTask(task, d))
+ event.fire(TaskInvalid(task, d), d)
+ bb.msg.error(bb.msg.domain.Build, "No such task: %s" % task)
+ return 1
+
+ quieterr = False
+ if d.getVarFlag(task, "quieterrors") is not None:
+ quieterr = True
try:
bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task)
@@ -292,6 +296,11 @@ def exec_task(task, d):
for func in postfuncs:
exec_func(func, localdata)
event.fire(TaskSucceeded(task, localdata), localdata)
+
+ # make stamp, or cause event and raise exception
+ if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d):
+ make_stamp(task, d)
+
except FuncFailed as message:
# Try to extract the optional logfile
try:
@@ -299,14 +308,22 @@ def exec_task(task, d):
except:
logfile = None
msg = message
- bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message )
- failedevent = TaskFailed(msg, logfile, task, d)
- event.fire(failedevent, d)
- raise EventException("Function failed in task: %s" % message, failedevent)
-
- # make stamp, or cause event and raise exception
- if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d):
- make_stamp(task, d)
+ if not quieterr:
+ bb.msg.error(bb.msg.domain.Build, "Task failed: %s" % message )
+ failedevent = TaskFailed(msg, logfile, task, d)
+ event.fire(failedevent, d)
+ return 1
+
+ except Exception:
+ from traceback import format_exc
+ if not quieterr:
+ bb.msg.error(bb.msg.domain.Build, "Build of %s failed" % (task))
+ bb.msg.error(bb.msg.domain.Build, format_exc())
+ failedevent = TaskFailed("Task Failed", None, task, d)
+ event.fire(failedevent, d)
+ return 1
+
+ return 0
def extract_stamp(d, fn):
"""
@@ -380,6 +397,7 @@ def add_tasks(tasklist, d):
getTask('rdeptask')
getTask('recrdeptask')
getTask('nostamp')
+ getTask('fakeroot')
task_deps['parents'][task] = []
for dep in flags['deps']:
dep = data.expand(dep, d)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index a1620b016..3a2595662 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -70,12 +70,16 @@ class BBCooker:
self.cache = None
self.bb_cache = None
- self.server = server.BitBakeServer(self)
+ if server:
+ self.server = server.BitBakeServer(self)
self.configuration = configuration
self.configuration.data = bb.data.init()
+ if not server:
+ bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)
+
bb.data.inheritFromOS(self.configuration.data)
self.parseConfigurationFiles(self.configuration.file)
@@ -544,7 +548,6 @@ class BBCooker:
bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
-
except IOError as e:
bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e)))
except bb.parse.ParseError as details:
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 636983edc..9e37f5e32 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -229,6 +229,15 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
for key in keys:
emit_var(key, o, d, all and not isfunc) and o.write('\n')
+def export_vars(d):
+ keys = (key for key in d.keys() if d.getVarFlag(key, "export"))
+ ret = {}
+ for k in keys:
+ v = d.getVar(k, True)
+ if v:
+ ret[k] = v
+ return ret
+
def update_data(d):
"""Performs final steps upon the datastore, including application of overrides"""
d.finalize()
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 7731649ef..f5ba6eab3 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -31,6 +31,7 @@ import pickle
# the runqueue forks off.
worker_pid = 0
worker_pipe = None
+useStdout = True
class Event:
"""Base class for events"""
@@ -102,15 +103,12 @@ def fire(event, d):
def worker_fire(event, d):
data = "<event>" + pickle.dumps(event) + "</event>"
- try:
- if os.write(worker_pipe, data) != len (data):
- print("Error sending event to server (short write)")
- except OSError:
- sys.exit(1)
+ worker_pipe.write(data)
+ worker_pipe.flush()
def fire_from_worker(event, d):
if not event.startswith("<event>") or not event.endswith("</event>"):
- print("Error, not an event")
+ print("Error, not an event %s" % event)
return
event = pickle.loads(event[7:-8])
fire_ui_handlers(event, d)
@@ -140,6 +138,7 @@ def remove(name, handler):
def register_UIHhandler(handler):
bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
_ui_handlers[_ui_handler_seq] = handler
+ bb.event.useStdout = False
return _ui_handler_seq
def unregister_UIHhandler(handlerNum):
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index 8d2bcce45..5bb30e8dd 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -109,7 +109,7 @@ def debug(level, msgdomain, msg, fn = None):
if debug_level[msgdomain] >= level:
bb.event.fire(MsgDebug(msg), None)
- if not bb.event._ui_handlers:
+ if bb.event.useStdout:
print('DEBUG: %s' % (msg))
def note(level, msgdomain, msg, fn = None):
@@ -118,17 +118,18 @@ def note(level, msgdomain, msg, fn = None):
if level == 1 or verbose or debug_level[msgdomain] >= 1:
bb.event.fire(MsgNote(msg), None)
- if not bb.event._ui_handlers:
+ if bb.event.useStdout:
print('NOTE: %s' % (msg))
def warn(msgdomain, msg, fn = None):
bb.event.fire(MsgWarn(msg), None)
- if not bb.event._ui_handlers:
+ if bb.event.useStdout:
print('WARNING: %s' % (msg))
def error(msgdomain, msg, fn = None):
bb.event.fire(MsgError(msg), None)
- print('ERROR: %s' % (msg))
+ if bb.event.useStdout:
+ print('ERROR: %s' % (msg))
def fatal(msgdomain, msg, fn = None):
bb.event.fire(MsgFatal(msg), None)
@@ -137,5 +138,5 @@ def fatal(msgdomain, msg, fn = None):
def plain(msg, fn = None):
bb.event.fire(MsgPlain(msg), None)
- if not bb.event._ui_handlers:
+ if bb.event.useStdout:
print(msg)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 9127f248d..b2d5fc01a 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -23,6 +23,7 @@ Handles preparation and execution of a queue of tasks
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import bb, os, sys
+import subprocess
from bb import msg, data, event
import signal
import stat
@@ -937,6 +938,7 @@ class RunQueueExecute:
self.runq_complete = []
self.build_pids = {}
self.build_pipes = {}
+ self.build_procs = {}
self.failed_fnids = []
def runqueue_process_waitpid(self):
@@ -944,19 +946,22 @@ class RunQueueExecute:
Return none is there are no processes awaiting result collection, otherwise
collect the process exit codes and close the information pipe.
"""
- result = os.waitpid(-1, os.WNOHANG)
- if result[0] is 0 and result[1] is 0:
- return None
- task = self.build_pids[result[0]]
- del self.build_pids[result[0]]
- self.build_pipes[result[0]].close()
- del self.build_pipes[result[0]]
- if result[1] != 0:
- self.task_fail(task, result[1])
- else:
- self.task_complete(task)
- self.stats.taskCompleted()
- bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
+ for pid in self.build_procs.keys():
+ proc = self.build_procs[pid]
+ proc.poll()
+ if proc.returncode is not None:
+ task = self.build_pids[pid]
+ del self.build_pids[pid]
+ self.build_pipes[pid].close()
+ del self.build_pipes[pid]
+ del self.build_procs[pid]
+ if proc.returncode != 0:
+ self.task_fail(task, proc.returncode)
+ else:
+ self.task_complete(task)
+ self.stats.taskCompleted()
+ bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
+
def finish_now(self):
if self.stats.active:
@@ -990,31 +995,8 @@ class RunQueueExecute:
def fork_off_task(self, fn, task, taskname):
sys.stdout.flush()
sys.stderr.flush()
- try:
- pipein, pipeout = os.pipe()
- pid = os.fork()
- except OSError as e:
- bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
- if pid == 0:
- os.close(pipein)
- # Save out the PID so that the event can include it the
- # events
- bb.event.worker_pid = os.getpid()
- bb.event.worker_pipe = pipeout
-
- self.rq.state = runQueueChildProcess
- # Make the child the process group leader
- os.setpgid(0, 0)
- # No stdin
- newsi = os.open('/dev/null', os.O_RDWR)
- os.dup2(newsi, sys.stdin.fileno())
- # Stdout to a logfile
- #logout = data.expand("${TMPDIR}/log/stdout.%s" % os.getpid(), self.cfgData, True)
- #mkdirhier(os.path.dirname(logout))
- #newso = open(logout, 'w')
- #os.dup2(newso.fileno(), sys.stdout.fileno())
- #os.dup2(newso.fileno(), sys.stderr.fileno())
+ try:
bb.event.fire(runQueueTaskStarted(task, self.stats, self.rq), self.cfgData)
bb.msg.note(1, bb.msg.domain.RunQueue,
"Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.stats.active + self.stats.failed + 1,
@@ -1022,26 +1004,25 @@ class RunQueueExecute:
task,
self.rqdata.get_user_idstring(task)))
- bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data)
- bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", fn, self.cooker.configuration.data)
- try:
- the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
-
- if not self.cooker.configuration.dry_run:
- bb.build.exec_task(taskname, the_data)
- os._exit(0)
-
- except bb.build.EventException as e:
- event = e.args[1]
- bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
- os._exit(1)
- except Exception:
- from traceback import format_exc
- bb.msg.error(bb.msg.domain.Build, "Build of %s %s failed" % (fn, taskname))
- bb.msg.error(bb.msg.domain.Build, format_exc())
- os._exit(1)
- os._exit(0)
- return pid, pipein, pipeout
+ the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
+
+ env = bb.data.export_vars(the_data)
+
+ taskdep = self.rqdata.dataCache.task_deps[fn]
+ if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
+ envvars = the_data.getVar("FAKEROOTENV", True).split()
+ for var in envvars:
+ comps = var.split("=")
+ env[comps[0]] = comps[1]
+
+ proc = subprocess.Popen(["bitbake-runtask", fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+ pipein = proc.stdout
+ pipeout = proc.stdin
+ pid = proc.pid
+ except OSError as e:
+ bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
+
+ return proc
class RunQueueExecuteTasks(RunQueueExecute):
def __init__(self, rq):
@@ -1153,10 +1134,11 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.task_skip(task)
continue
- pid, pipein, pipeout = self.fork_off_task(fn, task, taskname)
+ proc = self.fork_off_task(fn, task, taskname)
- self.build_pids[pid] = task
- self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData)
+ self.build_pids[proc.pid] = task
+ self.build_procs[proc.pid] = proc
+ self.build_pipes[proc.pid] = runQueuePipe(proc.stdout, proc.stdin, self.cfgData)
self.runq_running[task] = 1
self.stats.taskActive()
if self.stats.active < self.number_tasks:
@@ -1356,10 +1338,11 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.task_skip(task)
return True
- pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname)
+ proc = self.fork_off_task(fn, realtask, taskname)
- self.build_pids[pid] = task
- self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData)
+ self.build_pids[proc.pid] = task
+ self.build_procs[proc.pid] = proc
+ self.build_pipes[proc.pid] = runQueuePipe(proc.stdout, proc.stdin, self.cfgData)
self.runq_running[task] = 1
self.stats.taskActive()
if self.stats.active < self.number_tasks:
@@ -1384,7 +1367,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.rq.state = runQueueRunInit
return True
-
class TaskFailure(Exception):
"""
Exception raised when a task in a runqueue fails
@@ -1437,14 +1419,14 @@ class runQueueTaskCompleted(runQueueEvent):
runQueueEvent.__init__(self, task, stats, rq)
self.message = "Task %s completed (%s)" % (task, self.taskstring)
-def check_stamp_fn(fn, taskname, d):
- rq = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", d)
- fn = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", d)
- fnid = rq.rqdata.taskData.getfn_id(fn)
- taskid = rq.get_task_id(fnid, taskname)
- if taskid is not None:
- return rq.check_stamp_task(taskid)
- return None
+#def check_stamp_fn(fn, taskname, d):
+# rq = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", d)
+# fn = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", d)
+# fnid = rq.rqdata.taskData.getfn_id(fn)
+# taskid = rq.get_task_id(fnid, taskname)
+# if taskid is not None:
+# return rq.check_stamp_task(taskid)
+# return None
class runQueuePipe():
"""
@@ -1452,7 +1434,7 @@ class runQueuePipe():
"""
def __init__(self, pipein, pipeout, d):
self.fd = pipein
- os.close(pipeout)
+ pipeout.close()
fcntl.fcntl(self.fd, fcntl.F_SETFL, fcntl.fcntl(self.fd, fcntl.F_GETFL) | os.O_NONBLOCK)
self.queue = ""
self.d = d
@@ -1460,8 +1442,8 @@ class runQueuePipe():
def read(self):
start = len(self.queue)
try:
- self.queue = self.queue + os.read(self.fd, 1024)
- except OSError:
+ self.queue = self.queue + self.fd.read(1024)
+ except IOError:
pass
end = len(self.queue)
index = self.queue.find("</event>")
@@ -1476,4 +1458,4 @@ class runQueuePipe():
continue
if len(self.queue) > 0:
print("Warning, worker left partial message")
- os.close(self.fd)
+ self.fd.close()
--
cgit v1.2.3