diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
| -rw-r--r-- | bitbake/lib/bb/cooker.py | 182 |
1 files changed, 108 insertions, 74 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 2406dfe95..488bc610d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -22,11 +22,13 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import sys, os, getopt, glob, copy, os.path, re, time +from __future__ import print_function +import sys, os, glob, os.path, re, time +import sre_constants +from cStringIO import StringIO +from contextlib import closing import bb -from bb import utils, data, parse, event, cache, providers, taskdata, runqueue -from bb import command -import itertools, sre_constants +from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue class MultipleMatches(Exception): """ @@ -121,11 +123,11 @@ class BBCooker: self.commandlineAction = None if 'world' in self.configuration.pkgs_to_build: - bb.error("'world' is not a valid target for --environment.") + bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.") elif len(self.configuration.pkgs_to_build) > 1: - bb.error("Only one target can be used with the --environment option.") + bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.") elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0: - bb.error("No target should be used with the --environment and --buildfile options.") + bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.") elif len(self.configuration.pkgs_to_build) > 0: self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build] else: @@ -138,21 +140,18 @@ class BBCooker: self.commandlineAction = ["showVersions"] elif self.configuration.parse_only: self.commandlineAction = ["parseFiles"] - # FIXME - implement - #elif self.configuration.interactive: - # self.interactiveMode() elif self.configuration.dot_graph: if self.configuration.pkgs_to_build: self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.error("Please specify a package name for dependency graph generation.") + bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.") else: if self.configuration.pkgs_to_build: self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.error("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + bb.msg.error(bb.msg.domain.Build, "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") def runCommands(self, server, data, abort): """ @@ -174,14 +173,14 @@ class BBCooker: except bb.build.FuncFailed: bb.msg.error(bb.msg.domain.Build, "task stack execution failed") raise - except bb.build.EventException, e: + except bb.build.EventException as e: event = e.args[1] bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event)) raise def tryBuild(self, fn, task): """ - Build a provider and its dependencies. + Build a provider and its dependencies. build_depends is a list of previous build dependencies (not runtime) If build_depends is empty, we're dealing with a runtime depends """ @@ -206,7 +205,7 @@ class BBCooker: # Sort by priority for pn in pkg_pn: - (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status) + (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status) preferred_versions[pn] = (pref_ver, pref_file) latest_versions[pn] = (last_ver, last_file) @@ -260,27 +259,22 @@ class BBCooker: if fn: try: envdata = self.bb_cache.loadDataFull(fn, self.configuration.data) - except IOError, e: + except IOError as e: bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e)) raise - except Exception, e: + except Exception as e: bb.msg.error(bb.msg.domain.Parsing, "%s" % e) raise - class dummywrite: - def __init__(self): - self.writebuf = "" - def write(self, output): - self.writebuf = self.writebuf + output - # emit variables and shell functions try: data.update_data(envdata) - wb = dummywrite() - data.emit_env(wb, envdata, True) - bb.msg.plain(wb.writebuf) - except Exception, e: + with closing(StringIO()) as env: + data.emit_env(env, envdata, True) + bb.msg.plain(env.getvalue()) + except Exception as e: bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e) + # emit the metadata which isnt valid shell data.expandKeys(envdata) for e in envdata.keys(): @@ -315,7 +309,7 @@ class BBCooker: rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) rq.prepare_runqueue() - seen_fnids = [] + seen_fnids = [] depend_tree = {} depend_tree["depends"] = {} depend_tree["tdepends"] = {} @@ -352,7 +346,7 @@ class BBCooker: depend_tree["rdepends-pn"][pn] = [] for rdep in taskdata.rdepids[fnid]: - depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) rdepends = self.status.rundeps[fn] for package in rdepends: @@ -397,51 +391,51 @@ class BBCooker: # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn depends_file = file('pn-depends.dot', 'w' ) - print >> depends_file, "digraph depends {" + print("digraph depends {", file=depends_file) for pn in depgraph["pn"]: fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] - print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) for pn in depgraph["depends"]: for depend in depgraph["depends"][pn]: - print >> depends_file, '"%s" -> "%s"' % (pn, depend) + print('"%s" -> "%s"' % (pn, depend), file=depends_file) for pn in depgraph["rdepends-pn"]: for rdepend in depgraph["rdepends-pn"][pn]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend) - print >> depends_file, "}" + print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file) + print("}", file=depends_file) bb.msg.plain("PN dependencies saved to 'pn-depends.dot'") depends_file = file('package-depends.dot', 'w' ) - print >> depends_file, "digraph depends {" + print("digraph depends {", file=depends_file) for package in depgraph["packages"]: pn = depgraph["packages"][package]["pn"] fn = depgraph["packages"][package]["filename"] version = depgraph["packages"][package]["version"] if package == pn: - print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) else: - print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn) + print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file) for depend in depgraph["depends"][pn]: - print >> depends_file, '"%s" -> "%s"' % (package, depend) + print('"%s" -> "%s"' % (package, depend), file=depends_file) for package in depgraph["rdepends-pkg"]: for rdepend in depgraph["rdepends-pkg"][package]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) + print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) for package in depgraph["rrecs-pkg"]: for rdepend in depgraph["rrecs-pkg"][package]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) - print >> depends_file, "}" + print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) + print("}", file=depends_file) bb.msg.plain("Package dependencies saved to 'package-depends.dot'") tdepends_file = file('task-depends.dot', 'w' ) - print >> tdepends_file, "digraph depends {" + print("digraph depends {", file=tdepends_file) for task in depgraph["tdepends"]: (pn, taskname) = task.rsplit(".", 1) fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] - print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn) + print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file) for dep in depgraph["tdepends"][task]: - print >> tdepends_file, '"%s" -> "%s"' % (task, dep) - print >> tdepends_file, "}" + print('"%s" -> "%s"' % (task, dep), file=tdepends_file) + print("}", file=tdepends_file) bb.msg.plain("Task dependencies saved to 'task-depends.dot'") def buildDepgraph( self ): @@ -452,9 +446,12 @@ class BBCooker: bb.data.update_data(localdata) bb.data.expandKeys(localdata) + matched = set() def calc_bbfile_priority(filename): - for (regex, pri) in self.status.bbfile_config_priorities: + for _, _, regex, pri in self.status.bbfile_config_priorities: if regex.match(filename): + if not regex in matched: + matched.add(regex) return pri return 0 @@ -473,6 +470,11 @@ class BBCooker: for p in self.status.pkg_fn: self.status.bbfile_priority[p] = calc_bbfile_priority(p) + for collection, pattern, regex, _ in self.status.bbfile_config_priorities: + if not regex in matched: + bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" % + (collection, pattern)) + def buildWorldTargetList(self): """ Build package list for "bitbake world" @@ -505,31 +507,57 @@ class BBCooker: """Drop off into a shell""" try: from bb import shell - except ImportError, details: + except ImportError as details: bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details ) else: shell.start( self ) + def _findLayerConf(self): + path = os.getcwd() + while path != "/": + bblayers = os.path.join(path, "conf", "bblayers.conf") + if os.path.exists(bblayers): + return bblayers + + path, _ = os.path.split(path) + def parseConfigurationFiles(self, files): try: data = self.configuration.data for f in files: data = bb.parse.handle(f, data) - layerconf = os.path.join(os.getcwd(), "conf", "bblayers.conf") - if os.path.exists(layerconf): + layerconf = self._findLayerConf() + if layerconf: bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf) data = bb.parse.handle(layerconf, data) layers = (bb.data.getVar('BBLAYERS', data, True) or "").split() + data = bb.data.createCopy(data) for layer in layers: bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer) bb.data.setVar('LAYERDIR', layer, data) data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data) + # XXX: Hack, relies on the local keys of the datasmart + # instance being stored in the 'dict' attribute and makes + # assumptions about how variable expansion works, but + # there's no better way to force an expansion of a single + # variable across the datastore today, and this at least + # lets us reference LAYERDIR without having to immediately + # eval all our variables that use it. + for key in data.dict: + if key != "_data": + value = data.getVar(key, False) + if value and "${LAYERDIR}" in value: + data.setVar(key, value.replace("${LAYERDIR}", layer)) + bb.data.delVar('LAYERDIR', data) + if not data.getVar("BBPATH", True): + bb.fatal("The BBPATH variable is not set") + data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data) self.configuration.data = data @@ -541,16 +569,17 @@ class BBCooker: # Nomally we only register event handlers at the end of parsing .bb files # We register any handlers we've found so far here... - for var in data.getVar('__BBHANDLERS', self.configuration.data) or []: - bb.event.register(var,bb.data.getVar(var, self.configuration.data)) + for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []: + bb.event.register(var, bb.data.getVar(var, self.configuration.data)) bb.fetch.fetcher_init(self.configuration.data) bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) - except IOError, e: + + except IOError as e: bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e))) - except bb.parse.ParseError, details: + except bb.parse.ParseError as details: bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) ) def handleCollections( self, collections ): @@ -573,7 +602,7 @@ class BBCooker: continue try: pri = int(priority) - self.status.bbfile_config_priorities.append((cre, pri)) + self.status.bbfile_config_priorities.append((c, regex, cre, pri)) except ValueError: bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) @@ -582,8 +611,8 @@ class BBCooker: Setup any variables needed before starting a build """ if not bb.data.getVar("BUILDNAME", self.configuration.data): - bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data) - bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data) + bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data) + bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data) def matchFiles(self, buildfile): """ @@ -630,13 +659,19 @@ class BBCooker: if (task == None): task = self.configuration.cmd - fn = self.matchFile(buildfile) + self.bb_cache = bb.cache.init(self) + self.status = bb.cache.CacheData() + + (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile) + buildfile = self.matchFile(fn) + fn = self.bb_cache.realfn2virtual(buildfile, cls) + self.buildSetVars() # Load data into the cache for fn and parse the loaded cache data - self.bb_cache = bb.cache.init(self) - self.status = bb.cache.CacheData() - self.bb_cache.loadData(fn, self.configuration.data, self.status) + the_data = self.bb_cache.loadDataFull(fn, self.configuration.data) + self.bb_cache.setData(fn, buildfile, the_data) + self.bb_cache.handle_data(fn, self.status) # Tweak some variables item = self.bb_cache.getVar('PN', fn, True) @@ -675,8 +710,8 @@ class BBCooker: failures = 0 try: retval = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: - for fnid in fnids: + except runqueue.TaskFailure as exc: + for fnid in exc.args: bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) failures = failures + 1 retval = False @@ -711,8 +746,8 @@ class BBCooker: failures = 0 try: retval = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: - for fnid in fnids: + except runqueue.TaskFailure as exc: + for fnid in exc.args: bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) failures = failures + 1 retval = False @@ -769,10 +804,10 @@ class BBCooker: ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" self.status.ignored_dependencies = set(ignore.split()) - + for dep in self.configuration.extra_assume_provided: self.status.ignored_dependencies.add(dep) - + self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") @@ -810,7 +845,7 @@ class BBCooker: for f in contents: (root, ext) = os.path.splitext(f) if ext == ".bb": - bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f))) + bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f))) return bbfiles def find_bbfiles( self, path ): @@ -822,7 +857,7 @@ class BBCooker: for ignored in ('SCCS', 'CVS', '.svn'): if ignored in dirs: dirs.remove(ignored) - found += [join(dir,f) for f in files if f.endswith('.bb')] + found += [join(dir, f) for f in files if f.endswith('.bb')] return found @@ -906,9 +941,9 @@ class BBCooker: pout.close() else: self.server.serve_forever() - + bb.event.fire(CookerExit(), self.configuration.event_data) - + class CookerExit(bb.event.Event): """ Notify clients of the Cooker shutdown @@ -937,9 +972,9 @@ class CookerParser: self.pointer = 0 def parse_next(self): + cooker = self.cooker if self.pointer < len(self.filelist): f = self.filelist[self.pointer] - cooker = self.cooker try: fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status) @@ -951,7 +986,7 @@ class CookerParser: self.skipped += skipped self.virtuals += virtuals - except IOError, e: + except IOError as e: self.error += 1 cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e)) @@ -960,7 +995,7 @@ class CookerParser: cooker.bb_cache.remove(f) cooker.bb_cache.sync() raise - except Exception, e: + except Exception as e: self.error += 1 cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f)) @@ -978,4 +1013,3 @@ class CookerParser: raise ParsingErrorsFound return False return True - |