diff options
| -rw-r--r-- | bitbake/lib/bb/build.py | 352 | ||||
| -rw-r--r-- | bitbake/lib/bb/data_smart.py | 4 | ||||
| -rw-r--r-- | bitbake/lib/bb/utils.py | 11 |
3 files changed, 185 insertions, 182 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index 3f6bc875c..79fb1def9 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -31,9 +31,13 @@ import sys import logging import bb import bb.utils +import bb.process logger = logging.getLogger("BitBake.Build") +NULL = open('/dev/null', 'r') + + # When we execute a python function we'd like certain things # in all namespaces, hence we add them to __builtins__ # If we do not do this and use the exec globals, they will @@ -53,8 +57,8 @@ class FuncFailed(Exception): def __str__(self): if self.logfile and os.path.exists(self.logfile): - msg = "%s (see %s for further information)" % \ - (self.message, self.logfile) + msg = ("%s (see %s for further information)" % + (self.message, self.logfile)) else: msg = self.message return msg @@ -95,30 +99,33 @@ class TaskInvalid(TaskBase): super(TaskInvalid, self).__init__(task, metadata) self._message = "No such task '%s'" % task -# functions -def exec_func(func, d, dirs = None): +class tee(file): + def write(self, string): + logger.plain(string) + file.write(self, string) + + def __repr__(self): + return "<open[tee] file '{0}'>".format(self.name) + + +def exec_func(func, d, dirs = None, logfile = NULL): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: - bb.warn("Function %s doesn't exist" % func) + if body is None: + logger.warn("Function %s doesn't exist", func) return flags = data.getVarFlags(func, d) - for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot', 'task']: - if not item in flags: - flags[item] = None - - ispython = flags['python'] - - cleandirs = flags['cleandirs'] + cleandirs = flags.get('cleandirs') if cleandirs: for cdir in data.expand(cleandirs, d).split(): - os.system("rm -rf %s" % cdir) + bb.utils.remove(cdir, True) if dirs is None: - dirs = flags['dirs'] + dirs = flags.get('dirs') if dirs: dirs = data.expand(dirs, d).split() @@ -128,214 +135,185 @@ def exec_func(func, d, dirs = None): adir = dirs[-1] else: adir = data.getVar('B', d, 1) + if not os.path.exists(adir): + adir = None - # Save current directory - try: - prevdir = os.getcwd() - except OSError: - prevdir = data.getVar('TOPDIR', d, True) - - # Setup scriptfile - t = data.getVar('T', d, 1) - if not t: - raise SystemExit("T variable not set, unable to build") - bb.utils.mkdirhier(t) - runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) - logfile = d.getVar("BB_LOGFILE", True) + ispython = flags.get('python') + if flags.get('fakeroot') and not flags.get('task'): + bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func) - # Change to correct directory (if specified) - if adir and os.access(adir, os.F_OK): - os.chdir(adir) + tempdir = data.getVar('T', d, 1) + runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid())) locks = [] - lockfiles = flags['lockfiles'] + lockfiles = flags.get('lockfiles') if lockfiles: for lock in data.expand(lockfiles, d).split(): locks.append(bb.utils.lockfile(lock)) try: - # Run the function if ispython: - exec_func_python(func, d, runfile, logfile) + exec_func_python(func, d, runfile, logfile, cwd=adir) else: - exec_func_shell(func, d, runfile, logfile, flags) - - # Restore original directory - try: - os.chdir(prevdir) - except: - pass + exec_func_shell(func, d, runfile, logfile, cwd=adir) finally: - # Unlock any lockfiles for lock in locks: bb.utils.unlockfile(lock) -def exec_func_python(func, d, runfile, logfile): +_functionfmt = """ +def {function}(d): +{body} + +{function}(d) +""" +#logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") +def exec_func_python(func, d, runfile, logfile, cwd=None): """Execute a python BB 'function'""" - bbfile = bb.data.getVar('FILE', d, 1) - tmp = "def " + func + "(d):\n%s" % data.getVar(func, d) - tmp += '\n' + func + '(d)' + bbfile = d.getVar('FILE', True) + try: + olddir = os.getcwd() + except OSError: + olddir = None + code = _functionfmt.format(function=func, body=d.getVar(func, True)) + bb.utils.mkdirhier(os.path.dirname(runfile)) + with open(runfile, 'w') as script: + script.write(code) + + if cwd: + os.chdir(cwd) + + #handler = logging.StreamHandler(logfile) + #handler.setFormatter(logformatter) + #bblogger.addHandler(handler) - f = open(runfile, "w") - f.write(tmp) - comp = utils.better_compile(tmp, func, bbfile) try: - utils.better_exec(comp, {"d": d}, tmp, bbfile) + comp = utils.better_compile(code, func, bbfile) + utils.better_exec(comp, {"d": d}, code, bbfile) except: if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed): raise - raise FuncFailed(func, logfile) - - -def exec_func_shell(func, d, runfile, logfile, flags): - """Execute a shell BB 'function' Returns true if execution was successful. + raise FuncFailed(func, None) + finally: + #bblogger.removeHandler(handler) + if olddir: + os.chdir(olddir) - For this, it creates a bash shell script in the tmp dectory, writes the local - data into it and finally executes. The output of the shell will end in a log file and stdout. +def exec_func_shell(function, d, runfile, logfile, cwd=None): + """Execute a shell function from the metadata Note on directory behavior. The 'dirs' varflag should contain a list of the directories you need created prior to execution. The last item in the list is where we will chdir/cd to. """ - deps = flags['deps'] - check = flags['check'] - if check in globals(): - if globals()[check](func, deps): - return - - f = open(runfile, "w") - f.write("#!/bin/sh -e\n") - if logger.getEffectiveLevel() <= logging.DEBUG: - f.write("set -x\n") - data.emit_func(func, f, d) - - f.write("cd %s\n" % os.getcwd()) - if func: f.write("%s\n" % func) - f.close() - os.chmod(runfile, 0775) - if not func: - raise TypeError("Function argument must be a string") - - # execute function - if flags['fakeroot'] and not flags['task']: - bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func) - - lang_environment = "LC_ALL=C " - ret = os.system('%ssh -e %s' % (lang_environment, runfile)) + # Don't let the emitted shell script override PWD + d.delVarFlag('PWD', 'export') - if ret == 0: - return + with open(runfile, 'w') as script: + script.write('#!/bin/sh -e\n') + if logger.getEffectiveLevel() <= logging.DEBUG: + script.write("set -x\n") + data.emit_func(function, script, d) - raise FuncFailed(func, logfile) + script.write("%s\n" % function) + os.fchmod(script.fileno(), 0775) + env = { + 'PATH': d.getVar('PATH', True), + 'LC_ALL': 'C', + } -def exec_task(fn, task, d): - """Execute an BB 'task' + cmd = runfile - The primary difference between executing a task versus executing - a function is that a task exists in the task digraph, and therefore - has dependencies amongst other tasks.""" + if logger.getEffectiveLevel() <= logging.DEBUG: + logfile = LogTee(logger, logfile) - # Check whther this is a valid task + try: + bb.process.run(cmd, env=env, cwd=cwd, shell=False, stdin=NULL, + log=logfile) + except bb.process.CmdError: + raise FuncFailed(function, logfile.name) + +def _task_data(fn, task, d): + localdata = data.createCopy(d) + localdata.setVar('BB_FILENAME', fn) + localdata.setVar('BB_CURRENTTASK', task[3:]) + localdata.setVar('OVERRIDES', 'task-%s:%s' % + (task[3:], d.getVar('OVERRIDES', False))) + localdata.finalize() + data.expandKeys(localdata) + return localdata + +def _exec_task(fn, task, d, quieterr): + """Execute a BB 'task' + + Execution of a task involves a bit more setup than executing a function, + running it with its own local metadata, and with some useful variables set. + """ if not data.getVarFlag(task, 'task', d): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 - quieterr = False - if d.getVarFlag(task, "quieterrors") is not None: - quieterr = True + logger.debug(1, "Executing task %s", task) - try: - logger.debug(1, "Executing task %s", task) - old_overrides = data.getVar('OVERRIDES', d, 0) - localdata = data.createCopy(d) - data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) - data.update_data(localdata) - data.expandKeys(localdata) - data.setVar('BB_FILENAME', fn, d) - data.setVar('BB_CURRENTTASK', task[3:], d) - event.fire(TaskStarted(task, localdata), localdata) - - # Setup logfiles - t = data.getVar('T', d, 1) - if not t: - raise SystemExit("T variable not set, unable to build") - bb.utils.mkdirhier(t) - loglink = "%s/log.%s" % (t, task) - logfile = "%s/log.%s.%s" % (t, task, str(os.getpid())) - d.setVar("BB_LOGFILE", logfile) - - # Even though the log file has not yet been opened, lets create the link - if loglink: - try: - os.remove(loglink) - except OSError as e: - pass - - try: - os.symlink(logfile, loglink) - except OSError as e: - pass - - # Handle logfiles - si = file('/dev/null', 'r') - try: - so = file(logfile, 'w') - except OSError: - logger.exception("Opening log file '%s'", logfile) - pass - se = so + localdata = _task_data(fn, task, d) + tempdir = localdata.getVar('T', True) + if not tempdir: + bb.fatal("T variable not set, unable to build") - # Dup the existing fds so we dont lose them - osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] - oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] - ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] + bb.utils.mkdirhier(tempdir) + loglink = os.path.join(tempdir, 'log.{0}'.format(task)) + logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) + if loglink: + bb.utils.remove(loglink) - # Replace those fds with our own - os.dup2(si.fileno(), osi[1]) - os.dup2(so.fileno(), oso[1]) - os.dup2(se.fileno(), ose[1]) + try: + os.symlink(logfn, loglink) + except OSError: + pass - # Since we've remapped stdout and stderr, its safe for log messages to be printed there now - # exec_func can nest so we have to save state - origstdout = bb.event.useStdout - bb.event.useStdout = True + prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) + postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) + # Handle logfiles + si = file('/dev/null', 'r') + try: + logfile = file(logfn, 'w') + except OSError: + logger.exception("Opening log file '%s'", logfn) + pass - prefuncs = (data.getVarFlag(task, 'prefuncs', localdata) or "").split() - for func in prefuncs: - exec_func(func, localdata) - exec_func(task, localdata) - postfuncs = (data.getVarFlag(task, 'postfuncs', localdata) or "").split() - for func in postfuncs: - exec_func(func, localdata) + # Dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] - event.fire(TaskSucceeded(task, localdata), localdata) + # Replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(logfile.fileno(), oso[1]) + os.dup2(logfile.fileno(), ose[1]) - # make stamp, or cause event and raise exception - if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): - make_stamp(task, d) + # Since we've remapped stdout and stderr, its safe for log messages to be printed there now + # exec_func can nest so we have to save state + origstdout = bb.event.useStdout + bb.event.useStdout = True + event.fire(TaskStarted(task, localdata), localdata) + try: + for func in (prefuncs or '').split(): + exec_func(func, localdata, logfile=logfile) + exec_func(task, localdata, logfile=logfile) + for func in (postfuncs or '').split(): + exec_func(func, localdata, logfile=logfile) except FuncFailed as exc: if not quieterr: logger.error(str(exc)) - failedevent = TaskFailed(exc.name, exc.logfile, task, d) - event.fire(failedevent, d) - return 1 - - except Exception: - from traceback import format_exc - if not quieterr: - logger.error("Build of %s failed" % (task)) - logger.error(format_exc()) - failedevent = TaskFailed("Task Failed", None, task, d) - event.fire(failedevent, d) + event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata) return 1 finally: sys.stdout.flush() @@ -348,26 +326,40 @@ def exec_task(fn, task, d): os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) - # Close our logs - si.close() - so.close() - se.close() - - if logfile and os.path.exists(logfile) and os.path.getsize(logfile) == 0: - logger.debug(2, "Zero size logfile %s, removing", logfile) - os.remove(logfile) - try: - os.remove(loglink) - except OSError as e: - pass - # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) + si.close() + + logfile.close() + if os.path.exists(logfn) and os.path.getsize(logfn) == 0: + logger.debug(2, "Zero size logfn %s, removing", logfn) + bb.utils.remove(logfn) + bb.utils.remove(loglink) + event.fire(TaskSucceeded(task, localdata), localdata) + + if not d.getVarFlag(task, 'nostamp') and not d.getVarFlag(task, 'selfstamp'): + make_stamp(task, d) return 0 +def exec_task(fn, task, d): + try: + quieterr = False + if d.getVarFlag(task, "quieterrors") is not None: + quieterr = True + + return _exec_task(fn, task, d, quieterr) + except Exception: + from traceback import format_exc + if not quieterr: + logger.error("Build of %s failed" % (task)) + logger.error(format_exc()) + failedevent = TaskFailed("Task Failed", None, task, d) + event.fire(failedevent, d) + return 1 + def extract_stamp(d, fn): """ Extracts stamp format which is either a data dictionary (fn unset) diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 3f2d42c1b..16270461a 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -291,13 +291,13 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) self.dict[var][flag] = flagvalue - def getVarFlag(self, var, flag, exp = False): + def getVarFlag(self, var, flag, expand = False): local_var = self._findVar(var) value = None if local_var: if flag in local_var: value = copy.copy(local_var[flag]) - if exp and value: + if expand and value: value = self.expand(value, None) return value diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index e4c12fd1a..d9f543bc6 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -579,6 +579,17 @@ def build_environment(d): if export: os.environ[var] = bb.data.getVar(var, d, True) or "" +def remove(path, recurse=False): + """Equivalent to rm -f or rm -rf""" + import os, errno, shutil + try: + os.unlink(path) + except OSError, exc: + if recurse and exc.errno == errno.EISDIR: + shutil.rmtree(path) + elif exc.errno != errno.ENOENT: + raise + def prunedir(topdir): # Delete everything reachable from the directory named in 'topdir'. # CAUTION: This is dangerous! |