diff options
Diffstat (limited to 'bitbake/lib')
47 files changed, 1599 insertions, 1788 deletions
diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py index ca206cf4b..6917ec378 100644 --- a/bitbake/lib/bb/COW.py +++ b/bitbake/lib/bb/COW.py @@ -3,7 +3,7 @@ # # This is a copy on write dictionary and set which abuses classes to try and be nice and fast. # -# Copyright (C) 2006 Tim Amsell +# Copyright (C) 2006 Tim Amsell # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,29 +18,31 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -#Please Note: +#Please Note: # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. # Assign a file to __warn__ to get warnings about slow operations. # +from __future__ import print_function import copy import types -types.ImmutableTypes = tuple([ \ - types.BooleanType, \ - types.ComplexType, \ - types.FloatType, \ - types.IntType, \ - types.LongType, \ - types.NoneType, \ - types.TupleType, \ - frozenset] + \ - list(types.StringTypes)) +ImmutableTypes = ( + types.NoneType, + bool, + complex, + float, + int, + long, + tuple, + frozenset, + basestring +) MUTABLE = "__mutable__" class COWMeta(type): pass - + class COWDictMeta(COWMeta): __warn__ = False __hasmutable__ = False @@ -59,12 +61,12 @@ class COWDictMeta(COWMeta): __call__ = cow def __setitem__(cls, key, value): - if not isinstance(value, types.ImmutableTypes): + if not isinstance(value, ImmutableTypes): if not isinstance(value, COWMeta): cls.__hasmutable__ = True key += MUTABLE setattr(cls, key, value) - + def __getmutable__(cls, key, readonly=False): nkey = key + MUTABLE try: @@ -77,10 +79,10 @@ class COWDictMeta(COWMeta): return value if not cls.__warn__ is False and not isinstance(value, COWMeta): - print >> cls.__warn__, "Warning: Doing a copy because %s is a mutable type." % key + print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__) try: value = value.copy() - except AttributeError, e: + except AttributeError as e: value = copy.copy(value) setattr(cls, nkey, value) return value @@ -98,13 +100,13 @@ class COWDictMeta(COWMeta): value = getattr(cls, key) except AttributeError: value = cls.__getmutable__(key, readonly) - - # This is for values which have been deleted + + # This is for values which have been deleted if value is cls.__marker__: raise AttributeError("key %s does not exist." % key) return value - except AttributeError, e: + except AttributeError as e: if not default is cls.__getmarker__: return default @@ -118,6 +120,9 @@ class COWDictMeta(COWMeta): key += MUTABLE delattr(cls, key) + def __contains__(cls, key): + return cls.has_key(key) + def has_key(cls, key): value = cls.__getreadonly__(key, cls.__marker__) if value is cls.__marker__: @@ -127,7 +132,7 @@ class COWDictMeta(COWMeta): def iter(cls, type, readonly=False): for key in dir(cls): if key.startswith("__"): - continue + continue if key.endswith(MUTABLE): key = key[:-len(MUTABLE)] @@ -153,11 +158,11 @@ class COWDictMeta(COWMeta): return cls.iter("keys") def itervalues(cls, readonly=False): if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True." + print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("values", readonly) def iteritems(cls, readonly=False): if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True." + print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("items", readonly) class COWSetMeta(COWDictMeta): @@ -176,13 +181,13 @@ class COWSetMeta(COWDictMeta): def remove(cls, value): COWDictMeta.__delitem__(cls, repr(hash(value))) - + def __in__(cls, value): return COWDictMeta.has_key(repr(hash(value))) def iterkeys(cls): raise TypeError("sets don't have keys") - + def iteritems(cls): raise TypeError("sets don't have 'items'") @@ -199,120 +204,120 @@ if __name__ == "__main__": import sys COWDictBase.__warn__ = sys.stderr a = COWDictBase() - print "a", a + print("a", a) a['a'] = 'a' a['b'] = 'b' a['dict'] = {} b = a.copy() - print "b", b + print("b", b) b['c'] = 'b' - print + print() - print "a", a + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(): - print x - print + print(x) + print() b['dict']['a'] = 'b' b['a'] = 'c' - print "a", a + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(): - print x - print + print(x) + print() try: b['dict2'] - except KeyError, e: - print "Okay!" + except KeyError as e: + print("Okay!") a['set'] = COWSetBase() a['set'].add("o1") a['set'].add("o1") a['set'].add("o2") - print "a", a + print("a", a) for x in a['set'].itervalues(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b['set'].itervalues(): - print x - print + print(x) + print() b['set'].add('o3') - print "a", a + print("a", a) for x in a['set'].itervalues(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b['set'].itervalues(): - print x - print + print(x) + print() a['set2'] = set() a['set2'].add("o1") a['set2'].add("o1") a['set2'].add("o2") - print "a", a + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(readonly=True): - print x - print + print(x) + print() del b['b'] try: - print b['b'] + print(b['b']) except KeyError: - print "Yay! deleted key raises error" + print("Yay! deleted key raises error") if b.has_key('b'): - print "Boo!" + print("Boo!") else: - print "Yay - has_key with delete works!" - - print "a", a + print("Yay - has_key with delete works!") + + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(readonly=True): - print x - print + print(x) + print() b.__revertitem__('b') - print "a", a + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(readonly=True): - print x - print + print(x) + print() b.__revertitem__('dict') - print "a", a + print("a", a) for x in a.iteritems(): - print x - print "--" - print "b", b + print(x) + print("--") + print("b", b) for x in b.iteritems(readonly=True): - print x - print + print(x) + print() diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index c7cd0f62d..88adfc1df 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py @@ -21,39 +21,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = "1.9.0" - -__all__ = [ - - "debug", - "note", - "error", - "fatal", - - "mkdirhier", - "movefile", - "vercmp", - -# fetch - "decodeurl", - "encodeurl", - -# modules - "parse", - "data", - "command", - "event", - "build", - "fetch", - "manifest", - "methodpool", - "cache", - "runqueue", - "taskdata", - "providers", - ] - -import sys, os, types, re, string +__version__ = "1.11.0" + +import sys +if sys.version_info < (2, 6, 0): + raise RuntimeError("Sorry, python 2.6.0 or later is required for this version of bitbake") + +import os +import bb.msg if "BBDEBUG" in os.environ: level = int(os.environ["BBDEBUG"]) @@ -81,14 +56,45 @@ def fatal(*args): bb.msg.fatal(None, ''.join(args)) -# For compatibility -from bb.fetch import MalformedUrl, encodeurl, decodeurl -from bb.data import VarExpandError -from bb.utils import mkdirhier, movefile, copyfile, which -from bb.utils import vercmp +def deprecated(func, name = None, advice = ""): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emmitted + when the function is used.""" + import warnings + + if advice: + advice = ": %s" % advice + if name is None: + name = func.__name__ + + def newFunc(*args, **kwargs): + warnings.warn("Call to deprecated function %s%s." % (name, + advice), + category = PendingDeprecationWarning, + stacklevel = 2) + return func(*args, **kwargs) + newFunc.__name__ = func.__name__ + newFunc.__doc__ = func.__doc__ + newFunc.__dict__.update(func.__dict__) + return newFunc - -if __name__ == "__main__": - import doctest, bb - bb.msg.set_debug_level(0) - doctest.testmod(bb) +# For compatibility +def deprecate_import(current, modulename, fromlist, renames = None): + """Import objects from one module into another, wrapping them with a DeprecationWarning""" + import sys + + module = __import__(modulename, fromlist = fromlist) + for position, objname in enumerate(fromlist): + obj = getattr(module, objname) + newobj = deprecated(obj, "{0}.{1}".format(current, objname), + "Please use {0}.{1} instead".format(modulename, objname)) + if renames: + newname = renames[position] + else: + newname = objname + + setattr(sys.modules[current], newname, newobj) + +deprecate_import(__name__, "bb.fetch", ("MalformedUrl", "encodeurl", "decodeurl")) +deprecate_import(__name__, "bb.utils", ("mkdirhier", "movefile", "copyfile", "which")) +deprecate_import(__name__, "bb.utils", ["vercmp_string"], ["vercmp"]) diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index 43dbfc136..1f4107fb6 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -27,8 +27,9 @@ from bb import data, event, mkdirhier, utils import bb, os, sys +import bb.utils -# When we execute a python function we'd like certain things +# When we execute a python function we'd like certain things # in all namespaces, hence we add them to __builtins__ # If we do not do this and use the exec globals, they will # not be available to subfunctions. @@ -98,18 +99,19 @@ def exec_func(func, d, dirs = None): ispython = flags['python'] - cleandirs = (data.expand(flags['cleandirs'], d) or "").split() - for cdir in cleandirs: - os.system("rm -rf %s" % cdir) + cleandirs = flags['cleandirs'] + if cleandirs: + for cdir in data.expand(cleandirs, d).split(): + os.system("rm -rf %s" % cdir) - if dirs: - dirs = data.expand(dirs, d) - else: - dirs = (data.expand(flags['dirs'], d) or "").split() - for adir in dirs: - mkdirhier(adir) + if dirs is None: + dirs = flags['dirs'] + if dirs: + dirs = data.expand(dirs, d).split() - if len(dirs) > 0: + if dirs: + for adir in dirs: + bb.utils.mkdirhier(adir) adir = dirs[-1] else: adir = data.getVar('B', d, 1) @@ -123,8 +125,8 @@ def exec_func(func, d, dirs = None): # Setup logfiles t = data.getVar('T', d, 1) if not t: - bb.msg.fatal(bb.msg.domain.Build, "T not set") - mkdirhier(t) + raise SystemExit("T variable not set, unable to build") + bb.utils.mkdirhier(t) logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) @@ -139,7 +141,7 @@ def exec_func(func, d, dirs = None): so = os.popen("tee \"%s\"" % logfile, "w") else: so = file(logfile, 'w') - except OSError, e: + except OSError as e: bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) pass @@ -156,9 +158,10 @@ def exec_func(func, d, dirs = None): os.dup2(se.fileno(), ose[1]) locks = [] - lockfiles = (data.expand(flags['lockfiles'], d) or "").split() - for lock in lockfiles: - locks.append(bb.utils.lockfile(lock)) + lockfiles = flags['lockfiles'] + if lockfiles: + for lock in data.expand(lockfiles, d).split(): + locks.append(bb.utils.lockfile(lock)) try: # Run the function @@ -200,26 +203,22 @@ def exec_func(func, d, dirs = None): def exec_func_python(func, d, runfile, logfile): """Execute a python BB 'function'""" - import re, os bbfile = bb.data.getVar('FILE', d, 1) - tmp = "def " + func + "():\n%s" % data.getVar(func, d) - tmp += '\n' + func + '()' + tmp = "def " + func + "(d):\n%s" % data.getVar(func, d) + tmp += '\n' + func + '(d)' f = open(runfile, "w") f.write(tmp) comp = utils.better_compile(tmp, func, bbfile) - g = {} # globals - g['d'] = d try: - utils.better_exec(comp, g, tmp, bbfile) + utils.better_exec(comp, {"d": d}, tmp, bbfile) except: - (t,value,tb) = sys.exc_info() + (t, value, tb) = sys.exc_info() if t in [bb.parse.SkipPackage, bb.build.FuncFailed]: raise - bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func) - raise FuncFailed("function %s failed" % func, logfile) + raise FuncFailed("Function %s failed" % func, logfile) def exec_func_shell(func, d, runfile, logfile, flags): """Execute a shell BB 'function' Returns true if execution was successful. @@ -248,7 +247,6 @@ def exec_func_shell(func, d, runfile, logfile, flags): f.close() os.chmod(runfile, 0775) if not func: - bb.msg.error(bb.msg.domain.Build, "Function not specified") raise FuncFailed("Function not specified for exec_func_shell") # execute function @@ -262,7 +260,6 @@ def exec_func_shell(func, d, runfile, logfile, flags): if ret == 0: return - bb.msg.error(bb.msg.domain.Build, "Function %s failed" % func) raise FuncFailed("function %s failed" % func, logfile) @@ -287,7 +284,7 @@ def exec_task(task, d): event.fire(TaskStarted(task, localdata), localdata) exec_func(task, localdata) event.fire(TaskSucceeded(task, localdata), localdata) - except FuncFailed, message: + except FuncFailed as message: # Try to extract the optional logfile try: (msg, logfile) = message @@ -305,8 +302,8 @@ def exec_task(task, d): def extract_stamp(d, fn): """ - Extracts stamp format which is either a data dictonary (fn unset) - or a dataCache entry (fn set). + Extracts stamp format which is either a data dictonary (fn unset) + or a dataCache entry (fn set). """ if fn: return d.stamp[fn] @@ -323,7 +320,7 @@ def stamp_internal(task, d, file_name): if not stamp: return stamp = "%s.%s" % (stamp, task) - mkdirhier(os.path.dirname(stamp)) + bb.utils.mkdirhier(os.path.dirname(stamp)) # Remove the file and recreate to force timestamp # change on broken NFS filesystems if os.access(stamp, os.F_OK): @@ -363,7 +360,7 @@ def add_tasks(tasklist, d): if not task in task_deps['tasks']: task_deps['tasks'].append(task) - flags = data.getVarFlags(task, d) + flags = data.getVarFlags(task, d) def getTask(name): if not name in task_deps: task_deps[name] = {} @@ -389,4 +386,3 @@ def remove_task(task, kill, d): If kill is 1, also remove tasks that depend on this task.""" data.delVarFlag(task, 'task', d) - diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 1f180012e..da4546640 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -28,7 +28,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os, re +import os import bb.data import bb.utils @@ -44,10 +44,10 @@ class Cache: """ BitBake Cache implementation """ - def __init__(self, cooker): + def __init__(self, data): - self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) + self.cachedir = bb.data.getVar("CACHE", data, True) self.clean = {} self.checked = {} self.depends_cache = {} @@ -61,30 +61,28 @@ class Cache: return self.has_cache = True - self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") + self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) - try: - os.stat( self.cachedir ) - except OSError: - bb.mkdirhier( self.cachedir ) + bb.utils.mkdirhier(self.cachedir) # If any of configuration.data's dependencies are newer than the # cache there isn't even any point in loading it... newest_mtime = 0 - deps = bb.data.getVar("__depends", cooker.configuration.data, True) - for f,old_mtime in deps: - if old_mtime > newest_mtime: - newest_mtime = old_mtime + deps = bb.data.getVar("__depends", data) + + old_mtimes = [old_mtime for f, old_mtime in deps] + old_mtimes.append(newest_mtime) + newest_mtime = max(old_mtimes) if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: try: p = pickle.Unpickler(file(self.cachefile, "rb")) self.depends_cache, version_data = p.load() if version_data['CACHE_VER'] != __cache_version__: - raise ValueError, 'Cache Version Mismatch' + raise ValueError('Cache Version Mismatch') if version_data['BITBAKE_VER'] != bb.__version__: - raise ValueError, 'Bitbake Version Mismatch' + raise ValueError('Bitbake Version Mismatch') except EOFError: bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") self.depends_cache = {} @@ -92,27 +90,23 @@ class Cache: bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") self.depends_cache = {} else: - try: - os.stat( self.cachefile ) + if os.path.isfile(self.cachefile): bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") - except OSError: - pass def getVar(self, var, fn, exp = 0): """ Gets the value of a variable (similar to getVar in the data class) - + There are two scenarios: 1. We have cached data - serve from depends_cache[fn] - 2. We're learning what data to cache - serve from data + 2. We're learning what data to cache - serve from data backend but add a copy of the data to the cache. """ if fn in self.clean: return self.depends_cache[fn][var] - if not fn in self.depends_cache: - self.depends_cache[fn] = {} + self.depends_cache.setdefault(fn, {}) if fn != self.data_fn: # We're trying to access data in the cache which doesn't exist @@ -134,14 +128,14 @@ class Cache: self.data = data # Make sure __depends makes the depends_cache - # If we're a virtual class we need to make sure all our depends are appended + # If we're a virtual class we need to make sure all our depends are appended # to the depends of fn. - depends = self.getVar("__depends", virtualfn, True) or [] + depends = self.getVar("__depends", virtualfn) or set() + self.depends_cache.setdefault(fn, {}) if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: self.depends_cache[fn]["__depends"] = depends - for dep in depends: - if dep not in self.depends_cache[fn]["__depends"]: - self.depends_cache[fn]["__depends"].append(dep) + else: + self.depends_cache[fn]["__depends"].update(depends) # Make sure the variants always make it into the cache too self.getVar('__VARIANTS', virtualfn, True) @@ -217,7 +211,7 @@ class Cache: for data in bb_data: virtualfn = self.realfn2virtual(fn, data) self.setData(virtualfn, fn, bb_data[data]) - if self.getVar("__SKIPPED", virtualfn, True): + if self.getVar("__SKIPPED", virtualfn): skipped += 1 bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) else: @@ -258,11 +252,11 @@ class Cache: self.remove(fn) return False - mtime = bb.parse.cached_mtime_noerror(fn) + mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn) self.remove(fn) return False @@ -275,7 +269,7 @@ class Cache: # Check dependencies are still valid depends = self.getVar("__depends", fn, True) if depends: - for f,old_mtime in depends: + for f, old_mtime in depends: fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: @@ -345,14 +339,14 @@ class Cache: def handle_data(self, file_name, cacheData): """ - Save data we need into the cache + Save data we need into the cache """ pn = self.getVar('PN', file_name, True) pe = self.getVar('PE', file_name, True) or "0" pv = self.getVar('PV', file_name, True) if 'SRCREVINACTION' in pv: - bb.note("Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) + bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) pr = self.getVar('PR', file_name, True) dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") @@ -360,7 +354,7 @@ class Cache: packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() - cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True) + cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name) # build PackageName to FileName lookup table if pn not in cacheData.pkg_pn: @@ -371,7 +365,7 @@ class Cache: # build FileName to PackageName lookup table cacheData.pkg_fn[file_name] = pn - cacheData.pkg_pepvpr[file_name] = (pe,pv,pr) + cacheData.pkg_pepvpr[file_name] = (pe, pv, pr) cacheData.pkg_dp[file_name] = dp provides = [pn] @@ -400,13 +394,13 @@ class Cache: if not dep in cacheData.all_depends: cacheData.all_depends.append(dep) - # Build reverse hash for PACKAGES, so runtime dependencies + # Build reverse hash for PACKAGES, so runtime dependencies # can be be resolved (RDEPENDS, RRECOMMENDS etc.) for package in packages: if not package in cacheData.packages: cacheData.packages[package] = [] cacheData.packages[package].append(file_name) - rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() + rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() for package in packages_dynamic: if not package in cacheData.packages_dynamic: @@ -445,38 +439,45 @@ class Cache: self.getVar('__BB_DONT_CACHE', file_name, True) self.getVar('__VARIANTS', file_name, True) - def load_bbfile( self, bbfile , config): + def load_bbfile( self, bbfile, config): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped """ + chdir_back = False - import bb - from bb import utils, data, parse, debug, event, fatal + from bb import data, parse # expand tmpdir to include this topdir data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) oldpath = os.path.abspath(os.getcwd()) - if bb.parse.cached_mtime_noerror(bbfile_loc): - os.chdir(bbfile_loc) + parse.cached_mtime_noerror(bbfile_loc) bb_data = data.init_db(config) + # The ConfHandler first looks if there is a TOPDIR and if not + # then it would call getcwd(). + # Previously, we chdir()ed to bbfile_loc, called the handler + # and finally chdir()ed back, a couple of thousand times. We now + # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. + if not data.getVar('TOPDIR', bb_data): + chdir_back = True + data.setVar('TOPDIR', bbfile_loc, bb_data) try: bb_data = parse.handle(bbfile, bb_data) # read .bb data - os.chdir(oldpath) + if chdir_back: os.chdir(oldpath) return bb_data except: - os.chdir(oldpath) + if chdir_back: os.chdir(oldpath) raise def init(cooker): """ - The Objective: Cache the minimum amount of data possible yet get to the + The Objective: Cache the minimum amount of data possible yet get to the stage of building packages (i.e. tryBuild) without reparsing any .bb files. - To do this, we intercept getVar calls and only cache the variables we see - being accessed. We rely on the cache getVar calls being made for all - variables bitbake might need to use to reach this stage. For each cached + To do this, we intercept getVar calls and only cache the variables we see + being accessed. We rely on the cache getVar calls being made for all + variables bitbake might need to use to reach this stage. For each cached file we need to track: * Its mtime @@ -486,7 +487,7 @@ def init(cooker): Files causing parsing errors are evicted from the cache. """ - return Cache(cooker) + return Cache(cooker.configuration.data) diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py index 06bd203c9..9a8d689e2 100644 --- a/bitbake/lib/bb/command.py +++ b/bitbake/lib/bb/command.py @@ -20,7 +20,7 @@ Provide an interface to interact with the bitbake server through 'commands' # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. """ -The bitbake server takes 'commands' from its UI/commandline. +The bitbake server takes 'commands' from its UI/commandline. Commands are either synchronous or asynchronous. Async commands return data to the client in the form of events. Sync commands must only return data through the function return value @@ -62,7 +62,7 @@ class Command: try: command = commandline.pop(0) if command in CommandsSync.__dict__: - # Can run synchronous commands straight away + # Can run synchronous commands straight away return getattr(CommandsSync, command)(self.cmds_sync, self, commandline) if self.currentAsyncCommand is not None: return "Busy (%s in progress)" % self.currentAsyncCommand[0] @@ -89,7 +89,17 @@ class Command: return False else: return False - except: + except KeyboardInterrupt as exc: + self.finishAsyncCommand("Interrupted") + return False + except SystemExit as exc: + arg = exc.args[0] + if isinstance(arg, basestring): + self.finishAsyncCommand(arg) + else: + self.finishAsyncCommand("Exited with %s" % arg) + return False + except Exception: import traceback self.finishAsyncCommand(traceback.format_exc()) return False @@ -268,6 +278,3 @@ class CookerCommandSetExitCode(bb.event.Event): def __init__(self, exitcode): bb.event.Event.__init__(self) self.exitcode = int(exitcode) - - - diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 2406dfe95..488bc610d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -22,11 +22,13 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import sys, os, getopt, glob, copy, os.path, re, time +from __future__ import print_function +import sys, os, glob, os.path, re, time +import sre_constants +from cStringIO import StringIO +from contextlib import closing import bb -from bb import utils, data, parse, event, cache, providers, taskdata, runqueue -from bb import command -import itertools, sre_constants +from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue class MultipleMatches(Exception): """ @@ -121,11 +123,11 @@ class BBCooker: self.commandlineAction = None if 'world' in self.configuration.pkgs_to_build: - bb.error("'world' is not a valid target for --environment.") + bb.msg.error(bb.msg.domain.Build, "'world' is not a valid target for --environment.") elif len(self.configuration.pkgs_to_build) > 1: - bb.error("Only one target can be used with the --environment option.") + bb.msg.error(bb.msg.domain.Build, "Only one target can be used with the --environment option.") elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0: - bb.error("No target should be used with the --environment and --buildfile options.") + bb.msg.error(bb.msg.domain.Build, "No target should be used with the --environment and --buildfile options.") elif len(self.configuration.pkgs_to_build) > 0: self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build] else: @@ -138,21 +140,18 @@ class BBCooker: self.commandlineAction = ["showVersions"] elif self.configuration.parse_only: self.commandlineAction = ["parseFiles"] - # FIXME - implement - #elif self.configuration.interactive: - # self.interactiveMode() elif self.configuration.dot_graph: if self.configuration.pkgs_to_build: self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.error("Please specify a package name for dependency graph generation.") + bb.msg.error(bb.msg.domain.Build, "Please specify a package name for dependency graph generation.") else: if self.configuration.pkgs_to_build: self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd] else: self.commandlineAction = None - bb.error("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + bb.msg.error(bb.msg.domain.Build, "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") def runCommands(self, server, data, abort): """ @@ -174,14 +173,14 @@ class BBCooker: except bb.build.FuncFailed: bb.msg.error(bb.msg.domain.Build, "task stack execution failed") raise - except bb.build.EventException, e: + except bb.build.EventException as e: event = e.args[1] bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event)) raise def tryBuild(self, fn, task): """ - Build a provider and its dependencies. + Build a provider and its dependencies. build_depends is a list of previous build dependencies (not runtime) If build_depends is empty, we're dealing with a runtime depends """ @@ -206,7 +205,7 @@ class BBCooker: # Sort by priority for pn in pkg_pn: - (last_ver,last_file,pref_ver,pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status) + (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, self.configuration.data, self.status) preferred_versions[pn] = (pref_ver, pref_file) latest_versions[pn] = (last_ver, last_file) @@ -260,27 +259,22 @@ class BBCooker: if fn: try: envdata = self.bb_cache.loadDataFull(fn, self.configuration.data) - except IOError, e: + except IOError as e: bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e)) raise - except Exception, e: + except Exception as e: bb.msg.error(bb.msg.domain.Parsing, "%s" % e) raise - class dummywrite: - def __init__(self): - self.writebuf = "" - def write(self, output): - self.writebuf = self.writebuf + output - # emit variables and shell functions try: data.update_data(envdata) - wb = dummywrite() - data.emit_env(wb, envdata, True) - bb.msg.plain(wb.writebuf) - except Exception, e: + with closing(StringIO()) as env: + data.emit_env(env, envdata, True) + bb.msg.plain(env.getvalue()) + except Exception as e: bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e) + # emit the metadata which isnt valid shell data.expandKeys(envdata) for e in envdata.keys(): @@ -315,7 +309,7 @@ class BBCooker: rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) rq.prepare_runqueue() - seen_fnids = [] + seen_fnids = [] depend_tree = {} depend_tree["depends"] = {} depend_tree["tdepends"] = {} @@ -352,7 +346,7 @@ class BBCooker: depend_tree["rdepends-pn"][pn] = [] for rdep in taskdata.rdepids[fnid]: - depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) rdepends = self.status.rundeps[fn] for package in rdepends: @@ -397,51 +391,51 @@ class BBCooker: # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn depends_file = file('pn-depends.dot', 'w' ) - print >> depends_file, "digraph depends {" + print("digraph depends {", file=depends_file) for pn in depgraph["pn"]: fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] - print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) for pn in depgraph["depends"]: for depend in depgraph["depends"][pn]: - print >> depends_file, '"%s" -> "%s"' % (pn, depend) + print('"%s" -> "%s"' % (pn, depend), file=depends_file) for pn in depgraph["rdepends-pn"]: for rdepend in depgraph["rdepends-pn"][pn]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend) - print >> depends_file, "}" + print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file) + print("}", file=depends_file) bb.msg.plain("PN dependencies saved to 'pn-depends.dot'") depends_file = file('package-depends.dot', 'w' ) - print >> depends_file, "digraph depends {" + print("digraph depends {", file=depends_file) for package in depgraph["packages"]: pn = depgraph["packages"][package]["pn"] fn = depgraph["packages"][package]["filename"] version = depgraph["packages"][package]["version"] if package == pn: - print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file) else: - print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn) + print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file) for depend in depgraph["depends"][pn]: - print >> depends_file, '"%s" -> "%s"' % (package, depend) + print('"%s" -> "%s"' % (package, depend), file=depends_file) for package in depgraph["rdepends-pkg"]: for rdepend in depgraph["rdepends-pkg"][package]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) + print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) for package in depgraph["rrecs-pkg"]: for rdepend in depgraph["rrecs-pkg"][package]: - print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) - print >> depends_file, "}" + print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) + print("}", file=depends_file) bb.msg.plain("Package dependencies saved to 'package-depends.dot'") tdepends_file = file('task-depends.dot', 'w' ) - print >> tdepends_file, "digraph depends {" + print("digraph depends {", file=tdepends_file) for task in depgraph["tdepends"]: (pn, taskname) = task.rsplit(".", 1) fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] - print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn) + print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file) for dep in depgraph["tdepends"][task]: - print >> tdepends_file, '"%s" -> "%s"' % (task, dep) - print >> tdepends_file, "}" + print('"%s" -> "%s"' % (task, dep), file=tdepends_file) + print("}", file=tdepends_file) bb.msg.plain("Task dependencies saved to 'task-depends.dot'") def buildDepgraph( self ): @@ -452,9 +446,12 @@ class BBCooker: bb.data.update_data(localdata) bb.data.expandKeys(localdata) + matched = set() def calc_bbfile_priority(filename): - for (regex, pri) in self.status.bbfile_config_priorities: + for _, _, regex, pri in self.status.bbfile_config_priorities: if regex.match(filename): + if not regex in matched: + matched.add(regex) return pri return 0 @@ -473,6 +470,11 @@ class BBCooker: for p in self.status.pkg_fn: self.status.bbfile_priority[p] = calc_bbfile_priority(p) + for collection, pattern, regex, _ in self.status.bbfile_config_priorities: + if not regex in matched: + bb.msg.warn(bb.msg.domain.Provider, "No bb files matched BBFILE_PATTERN_%s '%s'" % + (collection, pattern)) + def buildWorldTargetList(self): """ Build package list for "bitbake world" @@ -505,31 +507,57 @@ class BBCooker: """Drop off into a shell""" try: from bb import shell - except ImportError, details: + except ImportError as details: bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details ) else: shell.start( self ) + def _findLayerConf(self): + path = os.getcwd() + while path != "/": + bblayers = os.path.join(path, "conf", "bblayers.conf") + if os.path.exists(bblayers): + return bblayers + + path, _ = os.path.split(path) + def parseConfigurationFiles(self, files): try: data = self.configuration.data for f in files: data = bb.parse.handle(f, data) - layerconf = os.path.join(os.getcwd(), "conf", "bblayers.conf") - if os.path.exists(layerconf): + layerconf = self._findLayerConf() + if layerconf: bb.msg.debug(2, bb.msg.domain.Parsing, "Found bblayers.conf (%s)" % layerconf) data = bb.parse.handle(layerconf, data) layers = (bb.data.getVar('BBLAYERS', data, True) or "").split() + data = bb.data.createCopy(data) for layer in layers: bb.msg.debug(2, bb.msg.domain.Parsing, "Adding layer %s" % layer) bb.data.setVar('LAYERDIR', layer, data) data = bb.parse.handle(os.path.join(layer, "conf", "layer.conf"), data) + # XXX: Hack, relies on the local keys of the datasmart + # instance being stored in the 'dict' attribute and makes + # assumptions about how variable expansion works, but + # there's no better way to force an expansion of a single + # variable across the datastore today, and this at least + # lets us reference LAYERDIR without having to immediately + # eval all our variables that use it. + for key in data.dict: + if key != "_data": + value = data.getVar(key, False) + if value and "${LAYERDIR}" in value: + data.setVar(key, value.replace("${LAYERDIR}", layer)) + bb.data.delVar('LAYERDIR', data) + if not data.getVar("BBPATH", True): + bb.fatal("The BBPATH variable is not set") + data = bb.parse.handle(os.path.join("conf", "bitbake.conf"), data) self.configuration.data = data @@ -541,16 +569,17 @@ class BBCooker: # Nomally we only register event handlers at the end of parsing .bb files # We register any handlers we've found so far here... - for var in data.getVar('__BBHANDLERS', self.configuration.data) or []: - bb.event.register(var,bb.data.getVar(var, self.configuration.data)) + for var in bb.data.getVar('__BBHANDLERS', self.configuration.data) or []: + bb.event.register(var, bb.data.getVar(var, self.configuration.data)) bb.fetch.fetcher_init(self.configuration.data) bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) - except IOError, e: + + except IOError as e: bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (files, str(e))) - except bb.parse.ParseError, details: + except bb.parse.ParseError as details: bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (files, details) ) def handleCollections( self, collections ): @@ -573,7 +602,7 @@ class BBCooker: continue try: pri = int(priority) - self.status.bbfile_config_priorities.append((cre, pri)) + self.status.bbfile_config_priorities.append((c, regex, cre, pri)) except ValueError: bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) @@ -582,8 +611,8 @@ class BBCooker: Setup any variables needed before starting a build """ if not bb.data.getVar("BUILDNAME", self.configuration.data): - bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data) - bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data) + bb.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'), self.configuration.data) + bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()), self.configuration.data) def matchFiles(self, buildfile): """ @@ -630,13 +659,19 @@ class BBCooker: if (task == None): task = self.configuration.cmd - fn = self.matchFile(buildfile) + self.bb_cache = bb.cache.init(self) + self.status = bb.cache.CacheData() + + (fn, cls) = self.bb_cache.virtualfn2realfn(buildfile) + buildfile = self.matchFile(fn) + fn = self.bb_cache.realfn2virtual(buildfile, cls) + self.buildSetVars() # Load data into the cache for fn and parse the loaded cache data - self.bb_cache = bb.cache.init(self) - self.status = bb.cache.CacheData() - self.bb_cache.loadData(fn, self.configuration.data, self.status) + the_data = self.bb_cache.loadDataFull(fn, self.configuration.data) + self.bb_cache.setData(fn, buildfile, the_data) + self.bb_cache.handle_data(fn, self.status) # Tweak some variables item = self.bb_cache.getVar('PN', fn, True) @@ -675,8 +710,8 @@ class BBCooker: failures = 0 try: retval = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: - for fnid in fnids: + except runqueue.TaskFailure as exc: + for fnid in exc.args: bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) failures = failures + 1 retval = False @@ -711,8 +746,8 @@ class BBCooker: failures = 0 try: retval = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: - for fnid in fnids: + except runqueue.TaskFailure as exc: + for fnid in exc.args: bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) failures = failures + 1 retval = False @@ -769,10 +804,10 @@ class BBCooker: ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" self.status.ignored_dependencies = set(ignore.split()) - + for dep in self.configuration.extra_assume_provided: self.status.ignored_dependencies.add(dep) - + self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") @@ -810,7 +845,7 @@ class BBCooker: for f in contents: (root, ext) = os.path.splitext(f) if ext == ".bb": - bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f))) + bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f))) return bbfiles def find_bbfiles( self, path ): @@ -822,7 +857,7 @@ class BBCooker: for ignored in ('SCCS', 'CVS', '.svn'): if ignored in dirs: dirs.remove(ignored) - found += [join(dir,f) for f in files if f.endswith('.bb')] + found += [join(dir, f) for f in files if f.endswith('.bb')] return found @@ -906,9 +941,9 @@ class BBCooker: pout.close() else: self.server.serve_forever() - + bb.event.fire(CookerExit(), self.configuration.event_data) - + class CookerExit(bb.event.Event): """ Notify clients of the Cooker shutdown @@ -937,9 +972,9 @@ class CookerParser: self.pointer = 0 def parse_next(self): + cooker = self.cooker if self.pointer < len(self.filelist): f = self.filelist[self.pointer] - cooker = self.cooker try: fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status) @@ -951,7 +986,7 @@ class CookerParser: self.skipped += skipped self.virtuals += virtuals - except IOError, e: + except IOError as e: self.error += 1 cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e)) @@ -960,7 +995,7 @@ class CookerParser: cooker.bb_cache.remove(f) cooker.bb_cache.sync() raise - except Exception, e: + except Exception as e: self.error += 1 cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f)) @@ -978,4 +1013,3 @@ class CookerParser: raise ParsingErrorsFound return False return True - diff --git a/bitbake/lib/bb/daemonize.py b/bitbake/lib/bb/daemonize.py index 1a8bb379f..f0714b3af 100644 --- a/bitbake/lib/bb/daemonize.py +++ b/bitbake/lib/bb/daemonize.py @@ -1,191 +1,190 @@ -"""
-Python Deamonizing helper
-
-Configurable daemon behaviors:
-
- 1.) The current working directory set to the "/" directory.
- 2.) The current file creation mode mask set to 0.
- 3.) Close all open files (1024).
- 4.) Redirect standard I/O streams to "/dev/null".
-
-A failed call to fork() now raises an exception.
-
-References:
- 1) Advanced Programming in the Unix Environment: W. Richard Stevens
- 2) Unix Programming Frequently Asked Questions:
- http://www.erlenstar.demon.co.uk/unix/faq_toc.html
-
-Modified to allow a function to be daemonized and return for
-bitbake use by Richard Purdie
-"""
-
-__author__ = "Chad J. Schroeder"
-__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
-__version__ = "0.2"
-
-# Standard Python modules.
-import os # Miscellaneous OS interfaces.
-import sys # System-specific parameters and functions.
-
-# Default daemon parameters.
-# File mode creation mask of the daemon.
-# For BitBake's children, we do want to inherit the parent umask.
-UMASK = None
-
-# Default maximum for the number of available file descriptors.
-MAXFD = 1024
-
-# The standard I/O file descriptors are redirected to /dev/null by default.
-if (hasattr(os, "devnull")):
- REDIRECT_TO = os.devnull
-else:
- REDIRECT_TO = "/dev/null"
-
-def createDaemon(function, logfile):
- """
- Detach a process from the controlling terminal and run it in the
- background as a daemon, returning control to the caller.
- """
-
- try:
- # Fork a child process so the parent can exit. This returns control to
- # the command-line or shell. It also guarantees that the child will not
- # be a process group leader, since the child receives a new process ID
- # and inherits the parent's process group ID. This step is required
- # to insure that the next call to os.setsid is successful.
- pid = os.fork()
- except OSError, e:
- raise Exception, "%s [%d]" % (e.strerror, e.errno)
-
- if (pid == 0): # The first child.
- # To become the session leader of this new session and the process group
- # leader of the new process group, we call os.setsid(). The process is
- # also guaranteed not to have a controlling terminal.
- os.setsid()
-
- # Is ignoring SIGHUP necessary?
- #
- # It's often suggested that the SIGHUP signal should be ignored before
- # the second fork to avoid premature termination of the process. The
- # reason is that when the first child terminates, all processes, e.g.
- # the second child, in the orphaned group will be sent a SIGHUP.
- #
- # "However, as part of the session management system, there are exactly
- # two cases where SIGHUP is sent on the death of a process:
- #
- # 1) When the process that dies is the session leader of a session that
- # is attached to a terminal device, SIGHUP is sent to all processes
- # in the foreground process group of that terminal device.
- # 2) When the death of a process causes a process group to become
- # orphaned, and one or more processes in the orphaned group are
- # stopped, then SIGHUP and SIGCONT are sent to all members of the
- # orphaned group." [2]
- #
- # The first case can be ignored since the child is guaranteed not to have
- # a controlling terminal. The second case isn't so easy to dismiss.
- # The process group is orphaned when the first child terminates and
- # POSIX.1 requires that every STOPPED process in an orphaned process
- # group be sent a SIGHUP signal followed by a SIGCONT signal. Since the
- # second child is not STOPPED though, we can safely forego ignoring the
- # SIGHUP signal. In any case, there are no ill-effects if it is ignored.
- #
- # import signal # Set handlers for asynchronous events.
- # signal.signal(signal.SIGHUP, signal.SIG_IGN)
-
- try:
- # Fork a second child and exit immediately to prevent zombies. This
- # causes the second child process to be orphaned, making the init
- # process responsible for its cleanup. And, since the first child is
- # a session leader without a controlling terminal, it's possible for
- # it to acquire one by opening a terminal in the future (System V-
- # based systems). This second fork guarantees that the child is no
- # longer a session leader, preventing the daemon from ever acquiring
- # a controlling terminal.
- pid = os.fork() # Fork a second child.
- except OSError, e:
- raise Exception, "%s [%d]" % (e.strerror, e.errno)
-
- if (pid == 0): # The second child.
- # We probably don't want the file mode creation mask inherited from
- # the parent, so we give the child complete control over permissions.
- if UMASK is not None:
- os.umask(UMASK)
- else:
- # Parent (the first child) of the second child.
- os._exit(0)
- else:
- # exit() or _exit()?
- # _exit is like exit(), but it doesn't call any functions registered
- # with atexit (and on_exit) or any registered signal handlers. It also
- # closes any open file descriptors. Using exit() may cause all stdio
- # streams to be flushed twice and any temporary files may be unexpectedly
- # removed. It's therefore recommended that child branches of a fork()
- # and the parent branch(es) of a daemon use _exit().
- return
-
- # Close all open file descriptors. This prevents the child from keeping
- # open any file descriptors inherited from the parent. There is a variety
- # of methods to accomplish this task. Three are listed below.
- #
- # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
- # number of open file descriptors to close. If it doesn't exists, use
- # the default value (configurable).
- #
- # try:
- # maxfd = os.sysconf("SC_OPEN_MAX")
- # except (AttributeError, ValueError):
- # maxfd = MAXFD
- #
- # OR
- #
- # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
- # maxfd = os.sysconf("SC_OPEN_MAX")
- # else:
- # maxfd = MAXFD
- #
- # OR
- #
- # Use the getrlimit method to retrieve the maximum file descriptor number
- # that can be opened by this process. If there is not limit on the
- # resource, use the default value.
- #
- import resource # Resource usage information.
- maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
- if (maxfd == resource.RLIM_INFINITY):
- maxfd = MAXFD
-
- # Iterate through and close all file descriptors.
-# for fd in range(0, maxfd):
-# try:
-# os.close(fd)
-# except OSError: # ERROR, fd wasn't open to begin with (ignored)
-# pass
-
- # Redirect the standard I/O file descriptors to the specified file. Since
- # the daemon has no controlling terminal, most daemons redirect stdin,
- # stdout, and stderr to /dev/null. This is done to prevent side-effects
- # from reads and writes to the standard I/O file descriptors.
-
- # This call to open is guaranteed to return the lowest file descriptor,
- # which will be 0 (stdin), since it was closed above.
-# os.open(REDIRECT_TO, os.O_RDWR) # standard input (0)
-
- # Duplicate standard input to standard output and standard error.
-# os.dup2(0, 1) # standard output (1)
-# os.dup2(0, 2) # standard error (2)
-
-
- si = file('/dev/null', 'r')
- so = file(logfile, 'w')
- se = so
-
-
- # Replace those fds with our own
- os.dup2(si.fileno(), sys.stdin.fileno())
- os.dup2(so.fileno(), sys.stdout.fileno())
- os.dup2(se.fileno(), sys.stderr.fileno())
-
- function()
-
- os._exit(0)
-
+""" +Python Deamonizing helper + +Configurable daemon behaviors: + + 1.) The current working directory set to the "/" directory. + 2.) The current file creation mode mask set to 0. + 3.) Close all open files (1024). + 4.) Redirect standard I/O streams to "/dev/null". + +A failed call to fork() now raises an exception. + +References: + 1) Advanced Programming in the Unix Environment: W. Richard Stevens + 2) Unix Programming Frequently Asked Questions: + http://www.erlenstar.demon.co.uk/unix/faq_toc.html + +Modified to allow a function to be daemonized and return for +bitbake use by Richard Purdie +""" + +__author__ = "Chad J. Schroeder" +__copyright__ = "Copyright (C) 2005 Chad J. Schroeder" +__version__ = "0.2" + +# Standard Python modules. +import os # Miscellaneous OS interfaces. +import sys # System-specific parameters and functions. + +# Default daemon parameters. +# File mode creation mask of the daemon. +# For BitBake's children, we do want to inherit the parent umask. +UMASK = None + +# Default maximum for the number of available file descriptors. +MAXFD = 1024 + +# The standard I/O file descriptors are redirected to /dev/null by default. +if (hasattr(os, "devnull")): + REDIRECT_TO = os.devnull +else: + REDIRECT_TO = "/dev/null" + +def createDaemon(function, logfile): + """ + Detach a process from the controlling terminal and run it in the + background as a daemon, returning control to the caller. + """ + + try: + # Fork a child process so the parent can exit. This returns control to + # the command-line or shell. It also guarantees that the child will not + # be a process group leader, since the child receives a new process ID + # and inherits the parent's process group ID. This step is required + # to insure that the next call to os.setsid is successful. + pid = os.fork() + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid == 0): # The first child. + # To become the session leader of this new session and the process group + # leader of the new process group, we call os.setsid(). The process is + # also guaranteed not to have a controlling terminal. + os.setsid() + + # Is ignoring SIGHUP necessary? + # + # It's often suggested that the SIGHUP signal should be ignored before + # the second fork to avoid premature termination of the process. The + # reason is that when the first child terminates, all processes, e.g. + # the second child, in the orphaned group will be sent a SIGHUP. + # + # "However, as part of the session management system, there are exactly + # two cases where SIGHUP is sent on the death of a process: + # + # 1) When the process that dies is the session leader of a session that + # is attached to a terminal device, SIGHUP is sent to all processes + # in the foreground process group of that terminal device. + # 2) When the death of a process causes a process group to become + # orphaned, and one or more processes in the orphaned group are + # stopped, then SIGHUP and SIGCONT are sent to all members of the + # orphaned group." [2] + # + # The first case can be ignored since the child is guaranteed not to have + # a controlling terminal. The second case isn't so easy to dismiss. + # The process group is orphaned when the first child terminates and + # POSIX.1 requires that every STOPPED process in an orphaned process + # group be sent a SIGHUP signal followed by a SIGCONT signal. Since the + # second child is not STOPPED though, we can safely forego ignoring the + # SIGHUP signal. In any case, there are no ill-effects if it is ignored. + # + # import signal # Set handlers for asynchronous events. + # signal.signal(signal.SIGHUP, signal.SIG_IGN) + + try: + # Fork a second child and exit immediately to prevent zombies. This + # causes the second child process to be orphaned, making the init + # process responsible for its cleanup. And, since the first child is + # a session leader without a controlling terminal, it's possible for + # it to acquire one by opening a terminal in the future (System V- + # based systems). This second fork guarantees that the child is no + # longer a session leader, preventing the daemon from ever acquiring + # a controlling terminal. + pid = os.fork() # Fork a second child. + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid == 0): # The second child. + # We probably don't want the file mode creation mask inherited from + # the parent, so we give the child complete control over permissions. + if UMASK is not None: + os.umask(UMASK) + else: + # Parent (the first child) of the second child. + os._exit(0) + else: + # exit() or _exit()? + # _exit is like exit(), but it doesn't call any functions registered + # with atexit (and on_exit) or any registered signal handlers. It also + # closes any open file descriptors. Using exit() may cause all stdio + # streams to be flushed twice and any temporary files may be unexpectedly + # removed. It's therefore recommended that child branches of a fork() + # and the parent branch(es) of a daemon use _exit(). + return + + # Close all open file descriptors. This prevents the child from keeping + # open any file descriptors inherited from the parent. There is a variety + # of methods to accomplish this task. Three are listed below. + # + # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum + # number of open file descriptors to close. If it doesn't exists, use + # the default value (configurable). + # + # try: + # maxfd = os.sysconf("SC_OPEN_MAX") + # except (AttributeError, ValueError): + # maxfd = MAXFD + # + # OR + # + # if (os.sysconf_names.has_key("SC_OPEN_MAX")): + # maxfd = os.sysconf("SC_OPEN_MAX") + # else: + # maxfd = MAXFD + # + # OR + # + # Use the getrlimit method to retrieve the maximum file descriptor number + # that can be opened by this process. If there is not limit on the + # resource, use the default value. + # + import resource # Resource usage information. + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if (maxfd == resource.RLIM_INFINITY): + maxfd = MAXFD + + # Iterate through and close all file descriptors. +# for fd in range(0, maxfd): +# try: +# os.close(fd) +# except OSError: # ERROR, fd wasn't open to begin with (ignored) +# pass + + # Redirect the standard I/O file descriptors to the specified file. Since + # the daemon has no controlling terminal, most daemons redirect stdin, + # stdout, and stderr to /dev/null. This is done to prevent side-effects + # from reads and writes to the standard I/O file descriptors. + + # This call to open is guaranteed to return the lowest file descriptor, + # which will be 0 (stdin), since it was closed above. +# os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) + + # Duplicate standard input to standard output and standard error. +# os.dup2(0, 1) # standard output (1) +# os.dup2(0, 2) # standard error (2) + + + si = file('/dev/null', 'r') + so = file(logfile, 'w') + se = so + + + # Replace those fds with our own + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + function() + + os._exit(0) diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index c3bb1a1f4..636983edc 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py @@ -11,7 +11,7 @@ operations. At night the cookie monster came by and suggested 'give me cookies on setting the variables and things will work out'. Taking this suggestion into account applying the skills from the not yet passed 'Entwurf und -Analyse von Algorithmen' lecture and the cookie +Analyse von Algorithmen' lecture and the cookie monster seems to be right. We will track setVar more carefully to have faster update_data and expandKeys operations. @@ -37,39 +37,40 @@ the speed is more critical here. # #Based on functions from the base bb module, Copyright 2003 Holger Schurig -import sys, os, re, types +import sys, os, re if sys.argv[0][-5:] == "pydoc": path = os.path.dirname(os.path.dirname(sys.argv[1])) else: path = os.path.dirname(os.path.dirname(sys.argv[0])) -sys.path.insert(0,path) +sys.path.insert(0, path) +from itertools import groupby from bb import data_smart import bb -class VarExpandError(Exception): - pass - _dict_type = data_smart.DataSmart def init(): + """Return a new object representing the Bitbake data""" return _dict_type() def init_db(parent = None): + """Return a new object representing the Bitbake data, + optionally based on an existing object""" if parent: return parent.createCopy() else: return _dict_type() def createCopy(source): - """Link the source set to the destination - If one does not find the value in the destination set, - search will go on to the source set to get the value. - Value from source are copy-on-write. i.e. any try to - modify one of them will end up putting the modified value - in the destination set. - """ - return source.createCopy() + """Link the source set to the destination + If one does not find the value in the destination set, + search will go on to the source set to get the value. + Value from source are copy-on-write. i.e. any try to + modify one of them will end up putting the modified value + in the destination set. + """ + return source.createCopy() def initVar(var, d): """Non-destructive var init for data structure""" @@ -77,91 +78,34 @@ def initVar(var, d): def setVar(var, value, d): - """Set a variable to a given value - - Example: - >>> d = init() - >>> setVar('TEST', 'testcontents', d) - >>> print getVar('TEST', d) - testcontents - """ - d.setVar(var,value) + """Set a variable to a given value""" + d.setVar(var, value) def getVar(var, d, exp = 0): - """Gets the value of a variable - - Example: - >>> d = init() - >>> setVar('TEST', 'testcontents', d) - >>> print getVar('TEST', d) - testcontents - """ - return d.getVar(var,exp) + """Gets the value of a variable""" + return d.getVar(var, exp) def renameVar(key, newkey, d): - """Renames a variable from key to newkey - - Example: - >>> d = init() - >>> setVar('TEST', 'testcontents', d) - >>> renameVar('TEST', 'TEST2', d) - >>> print getVar('TEST2', d) - testcontents - """ + """Renames a variable from key to newkey""" d.renameVar(key, newkey) def delVar(var, d): - """Removes a variable from the data set - - Example: - >>> d = init() - >>> setVar('TEST', 'testcontents', d) - >>> print getVar('TEST', d) - testcontents - >>> delVar('TEST', d) - >>> print getVar('TEST', d) - None - """ + """Removes a variable from the data set""" d.delVar(var) def setVarFlag(var, flag, flagvalue, d): - """Set a flag for a given variable to a given value - - Example: - >>> d = init() - >>> setVarFlag('TEST', 'python', 1, d) - >>> print getVarFlag('TEST', 'python', d) - 1 - """ - d.setVarFlag(var,flag,flagvalue) + """Set a flag for a given variable to a given value""" + d.setVarFlag(var, flag, flagvalue) def getVarFlag(var, flag, d): - """Gets given flag from given var - - Example: - >>> d = init() - >>> setVarFlag('TEST', 'python', 1, d) - >>> print getVarFlag('TEST', 'python', d) - 1 - """ - return d.getVarFlag(var,flag) + """Gets given flag from given var""" + return d.getVarFlag(var, flag) def delVarFlag(var, flag, d): - """Removes a given flag from the variable's flags - - Example: - >>> d = init() - >>> setVarFlag('TEST', 'testflag', 1, d) - >>> print getVarFlag('TEST', 'testflag', d) - 1 - >>> delVarFlag('TEST', 'testflag', d) - >>> print getVarFlag('TEST', 'testflag', d) - None - - """ - d.delVarFlag(var,flag) + """Removes a given flag from the variable's flags""" + d.delVarFlag(var, flag) def setVarFlags(var, flags, d): """Set the flags for a given variable @@ -170,115 +114,27 @@ def setVarFlags(var, flags, d): setVarFlags will not clear previous flags. Think of this method as addVarFlags - - Example: - >>> d = init() - >>> myflags = {} - >>> myflags['test'] = 'blah' - >>> setVarFlags('TEST', myflags, d) - >>> print getVarFlag('TEST', 'test', d) - blah """ - d.setVarFlags(var,flags) + d.setVarFlags(var, flags) def getVarFlags(var, d): - """Gets a variable's flags - - Example: - >>> d = init() - >>> setVarFlag('TEST', 'test', 'blah', d) - >>> print getVarFlags('TEST', d)['test'] - blah - """ + """Gets a variable's flags""" return d.getVarFlags(var) def delVarFlags(var, d): - """Removes a variable's flags - - Example: - >>> data = init() - >>> setVarFlag('TEST', 'testflag', 1, data) - >>> print getVarFlag('TEST', 'testflag', data) - 1 - >>> delVarFlags('TEST', data) - >>> print getVarFlags('TEST', data) - None - - """ + """Removes a variable's flags""" d.delVarFlags(var) def keys(d): - """Return a list of keys in d - - Example: - >>> d = init() - >>> setVar('TEST', 1, d) - >>> setVar('MOO' , 2, d) - >>> setVarFlag('TEST', 'test', 1, d) - >>> keys(d) - ['TEST', 'MOO'] - """ + """Return a list of keys in d""" return d.keys() -def getData(d): - """Returns the data object used""" - return d - -def setData(newData, d): - """Sets the data object to the supplied value""" - d = newData - - -## -## Cookie Monsters' query functions -## -def _get_override_vars(d, override): - """ - Internal!!! - - Get the Names of Variables that have a specific - override. This function returns a iterable - Set or an empty list - """ - return [] - -def _get_var_flags_triple(d): - """ - Internal!!! - - """ - return [] __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") def expand(s, d, varname = None): - """Variable expansion using the data store. - - Example: - Standard expansion: - >>> d = init() - >>> setVar('A', 'sshd', d) - >>> print expand('/usr/bin/${A}', d) - /usr/bin/sshd - - Python expansion: - >>> d = init() - >>> print expand('result: ${@37 * 72}', d) - result: 2664 - - Shell expansion: - >>> d = init() - >>> print expand('${TARGET_MOO}', d) - ${TARGET_MOO} - >>> setVar('TARGET_MOO', 'yupp', d) - >>> print expand('${TARGET_MOO}',d) - yupp - >>> setVar('SRC_URI', 'http://somebug.${TARGET_MOO}', d) - >>> delVar('TARGET_MOO', d) - >>> print expand('${SRC_URI}', d) - http://somebug.${TARGET_MOO} - """ + """Variable expansion using the data store""" return d.expand(s, varname) def expandKeys(alterdata, readdata = None): @@ -295,38 +151,13 @@ def expandKeys(alterdata, readdata = None): continue todolist[key] = ekey - # These two for loops are split for performance to maximise the + # These two for loops are split for performance to maximise the # usefulness of the expand cache for key in todolist: ekey = todolist[key] renameVar(key, ekey, alterdata) -def expandData(alterdata, readdata = None): - """For each variable in alterdata, expand it, and update the var contents. - Replacements use data from readdata. - - Example: - >>> a=init() - >>> b=init() - >>> setVar("dlmsg", "dl_dir is ${DL_DIR}", a) - >>> setVar("DL_DIR", "/path/to/whatever", b) - >>> expandData(a, b) - >>> print getVar("dlmsg", a) - dl_dir is /path/to/whatever - """ - if readdata == None: - readdata = alterdata - - for key in keys(alterdata): - val = getVar(key, alterdata) - if type(val) is not types.StringType: - continue - expanded = expand(val, readdata) -# print "key is %s, val is %s, expanded is %s" % (key, val, expanded) - if val != expanded: - setVar(key, expanded, alterdata) - def inheritFromOS(d): """Inherit variables from the environment.""" for s in os.environ.keys(): @@ -351,21 +182,15 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False): if all: oval = getVar(var, d, 0) val = getVar(var, d, 1) - except KeyboardInterrupt: + except (KeyboardInterrupt, bb.build.FuncFailed): raise - except: - excname = str(sys.exc_info()[0]) - if excname == "bb.build.FuncFailed": - raise - o.write('# expansion of %s threw %s\n' % (var, excname)) + except Exception, exc: + o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc))) return 0 if all: o.write('# %s=%s\n' % (var, oval)) - if type(val) is not types.StringType: - return 0 - if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: return 0 @@ -375,10 +200,11 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False): o.write('unset %s\n' % varExpanded) return 1 - val.rstrip() if not val: return 0 + val = str(val) + if func: # NOTE: should probably check for unbalanced {} within the var o.write("%s() {\n%s\n}\n" % (varExpanded, val)) @@ -393,173 +219,22 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False): o.write('%s="%s"\n' % (varExpanded, alter)) return 1 - def emit_env(o=sys.__stdout__, d = init(), all=False): """Emits all items in the data store in a format such that it can be sourced by a shell.""" - env = keys(d) - - for e in env: - if getVarFlag(e, "func", d): - continue - emit_var(e, o, d, all) and o.write('\n') - - for e in env: - if not getVarFlag(e, "func", d): - continue - emit_var(e, o, d) and o.write('\n') + isfunc = lambda key: bool(d.getVarFlag(key, "func")) + keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc) + grouped = groupby(keys, isfunc) + for isfunc, keys in grouped: + for key in keys: + emit_var(key, o, d, all and not isfunc) and o.write('\n') def update_data(d): - """Modifies the environment vars according to local overrides and commands. - Examples: - Appending to a variable: - >>> d = init() - >>> setVar('TEST', 'this is a', d) - >>> setVar('TEST_append', ' test', d) - >>> setVar('TEST_append', ' of the emergency broadcast system.', d) - >>> update_data(d) - >>> print getVar('TEST', d) - this is a test of the emergency broadcast system. - - Prepending to a variable: - >>> setVar('TEST', 'virtual/libc', d) - >>> setVar('TEST_prepend', 'virtual/tmake ', d) - >>> setVar('TEST_prepend', 'virtual/patcher ', d) - >>> update_data(d) - >>> print getVar('TEST', d) - virtual/patcher virtual/tmake virtual/libc - - Overrides: - >>> setVar('TEST_arm', 'target', d) - >>> setVar('TEST_ramses', 'machine', d) - >>> setVar('TEST_local', 'local', d) - >>> setVar('OVERRIDES', 'arm', d) - - >>> setVar('TEST', 'original', d) - >>> update_data(d) - >>> print getVar('TEST', d) - target - - >>> setVar('OVERRIDES', 'arm:ramses:local', d) - >>> setVar('TEST', 'original', d) - >>> update_data(d) - >>> print getVar('TEST', d) - local - - CopyMonster: - >>> e = d.createCopy() - >>> setVar('TEST_foo', 'foo', e) - >>> update_data(e) - >>> print getVar('TEST', e) - local - - >>> setVar('OVERRIDES', 'arm:ramses:local:foo', e) - >>> update_data(e) - >>> print getVar('TEST', e) - foo - - >>> f = d.createCopy() - >>> setVar('TEST_moo', 'something', f) - >>> setVar('OVERRIDES', 'moo:arm:ramses:local:foo', e) - >>> update_data(e) - >>> print getVar('TEST', e) - foo - - - >>> h = init() - >>> setVar('SRC_URI', 'file://append.foo;patch=1 ', h) - >>> g = h.createCopy() - >>> setVar('SRC_URI_append_arm', 'file://other.foo;patch=1', g) - >>> setVar('OVERRIDES', 'arm:moo', g) - >>> update_data(g) - >>> print getVar('SRC_URI', g) - file://append.foo;patch=1 file://other.foo;patch=1 - - """ - bb.msg.debug(2, bb.msg.domain.Data, "update_data()") - - # now ask the cookie monster for help - #print "Cookie Monster" - #print "Append/Prepend %s" % d._special_values - #print "Overrides %s" % d._seen_overrides - - overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] - - # - # Well let us see what breaks here. We used to iterate - # over each variable and apply the override and then - # do the line expanding. - # If we have bad luck - which we will have - the keys - # where in some order that is so important for this - # method which we don't have anymore. - # Anyway we will fix that and write test cases this - # time. - - # - # First we apply all overrides - # Then we will handle _append and _prepend - # - - for o in overrides: - # calculate '_'+override - l = len(o)+1 - - # see if one should even try - if not d._seen_overrides.has_key(o): - continue - - vars = d._seen_overrides[o] - for var in vars: - name = var[:-l] - try: - d[name] = d[var] - except: - bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar") - - # now on to the appends and prepends - if d._special_values.has_key('_append'): - appends = d._special_values['_append'] or [] - for append in appends: - for (a, o) in getVarFlag(append, '_append', d) or []: - # maybe the OVERRIDE was not yet added so keep the append - if (o and o in overrides) or not o: - delVarFlag(append, '_append', d) - if o and not o in overrides: - continue - - sval = getVar(append,d) or "" - sval+=a - setVar(append, sval, d) - - - if d._special_values.has_key('_prepend'): - prepends = d._special_values['_prepend'] or [] - - for prepend in prepends: - for (a, o) in getVarFlag(prepend, '_prepend', d) or []: - # maybe the OVERRIDE was not yet added so keep the prepend - if (o and o in overrides) or not o: - delVarFlag(prepend, '_prepend', d) - if o and not o in overrides: - continue - - sval = a + (getVar(prepend,d) or "") - setVar(prepend, sval, d) - + """Performs final steps upon the datastore, including application of overrides""" + d.finalize() def inherits_class(klass, d): val = getVar('__inherit_cache', d) or [] if os.path.join('classes', '%s.bbclass' % klass) in val: return True return False - -def _test(): - """Start a doctest run on this module""" - import doctest - import bb - from bb import data - bb.msg.set_debug_level(0) - doctest.testmod(data) - -if __name__ == "__main__": - _test() diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 9067d54bf..01a333024 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -28,22 +28,16 @@ BitBake build tools. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, os, re, sys, time, types +import copy, re, sys import bb -from bb import utils, methodpool -from COW import COWDictBase -from new import classobj +from bb import utils +from bb.COW import COWDictBase -__setvar_keyword__ = ["_append","_prepend"] +__setvar_keyword__ = ["_append", "_prepend"] __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?') __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") -_expand_globals = { - "os": os, - "bb": bb, - "time": time, -} class DataSmart: @@ -55,9 +49,8 @@ class DataSmart: self._seen_overrides = seen self.expand_cache = {} - self.expand_locals = {"d": self} - def expand(self,s, varname): + def expand(self, s, varname): def var_sub(match): key = match.group()[2:-1] if varname and key: @@ -70,13 +63,12 @@ class DataSmart: return match.group() def python_sub(match): - import bb code = match.group()[3:-1] - s = eval(code, _expand_globals, self.expand_locals) - if type(s) == types.IntType: s = str(s) - return s + codeobj = compile(code.strip(), varname or "<expansion>", "eval") + value = utils.better_eval(codeobj, {"d": self}) + return str(value) - if type(s) is not types.StringType: # sanity check + if not isinstance(s, basestring): # sanity check return s if varname and varname in self.expand_cache: @@ -87,9 +79,8 @@ class DataSmart: try: s = __expand_var_regexp__.sub(var_sub, s) s = __expand_python_regexp__.sub(python_sub, s) - if s == olds: break - if type(s) is not types.StringType: # sanity check - bb.msg.error(bb.msg.domain.Data, 'expansion of %s returned non-string %s' % (olds, s)) + if s == olds: + break except KeyboardInterrupt: raise except: @@ -101,23 +92,86 @@ class DataSmart: return s + def finalize(self): + """Performs final steps upon the datastore, including application of overrides""" + + overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + + # + # Well let us see what breaks here. We used to iterate + # over each variable and apply the override and then + # do the line expanding. + # If we have bad luck - which we will have - the keys + # where in some order that is so important for this + # method which we don't have anymore. + # Anyway we will fix that and write test cases this + # time. + + # + # First we apply all overrides + # Then we will handle _append and _prepend + # + + for o in overrides: + # calculate '_'+override + l = len(o) + 1 + + # see if one should even try + if o not in self._seen_overrides: + continue + + vars = self._seen_overrides[o] + for var in vars: + name = var[:-l] + try: + self[name] = self[var] + except Exception: + bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar") + + # now on to the appends and prepends + if "_append" in self._special_values: + appends = self._special_values["_append"] or [] + for append in appends: + for (a, o) in self.getVarFlag(append, "_append") or []: + # maybe the OVERRIDE was not yet added so keep the append + if (o and o in overrides) or not o: + self.delVarFlag(append, "_append") + if o and not o in overrides: + continue + + sval = self.getVar(append, False) or "" + sval += a + self.setVar(append, sval) + + + if "_prepend" in self._special_values: + prepends = self._special_values["_prepend"] or [] + + for prepend in prepends: + for (a, o) in self.getVarFlag(prepend, "_prepend") or []: + # maybe the OVERRIDE was not yet added so keep the prepend + if (o and o in overrides) or not o: + self.delVarFlag(prepend, "_prepend") + if o and not o in overrides: + continue + + sval = a + (self.getVar(prepend, False) or "") + self.setVar(prepend, sval) + def initVar(self, var): self.expand_cache = {} if not var in self.dict: self.dict[var] = {} - def _findVar(self,var): - _dest = self.dict + def _findVar(self, var): + dest = self.dict + while dest: + if var in dest: + return dest[var] - while (_dest and var not in _dest): - if not "_data" in _dest: - _dest = None + if "_data" not in dest: break - _dest = _dest["_data"] - - if _dest and var in _dest: - return _dest[var] - return None + dest = dest["_data"] def _makeShadowCopy(self, var): if var in self.dict: @@ -130,7 +184,7 @@ class DataSmart: else: self.initVar(var) - def setVar(self,var,value): + def setVar(self, var, value): self.expand_cache = {} match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: @@ -145,7 +199,7 @@ class DataSmart: # pay the cookie monster try: self._special_values[keyword].add( base ) - except: + except KeyError: self._special_values[keyword] = set() self._special_values[keyword].add( base ) @@ -157,23 +211,23 @@ class DataSmart: # more cookies for the cookie monster if '_' in var: override = var[var.rfind('_')+1:] - if not self._seen_overrides.has_key(override): + if override not in self._seen_overrides: self._seen_overrides[override] = set() self._seen_overrides[override].add( var ) # setting var self.dict[var]["content"] = value - def getVar(self,var,exp): - value = self.getVarFlag(var,"content") + def getVar(self, var, exp): + value = self.getVarFlag(var, "content") if exp and value: - return self.expand(value,var) + return self.expand(value, var) return value def renameVar(self, key, newkey): """ - Rename the variable key to newkey + Rename the variable key to newkey """ val = self.getVar(key, 0) if val is not None: @@ -187,30 +241,30 @@ class DataSmart: dest = self.getVarFlag(newkey, i) or [] dest.extend(src) self.setVarFlag(newkey, i, dest) - - if self._special_values.has_key(i) and key in self._special_values[i]: + + if i in self._special_values and key in self._special_values[i]: self._special_values[i].remove(key) self._special_values[i].add(newkey) self.delVar(key) - def delVar(self,var): + def delVar(self, var): self.expand_cache = {} self.dict[var] = {} - def setVarFlag(self,var,flag,flagvalue): + def setVarFlag(self, var, flag, flagvalue): if not var in self.dict: self._makeShadowCopy(var) self.dict[var][flag] = flagvalue - def getVarFlag(self,var,flag): + def getVarFlag(self, var, flag): local_var = self._findVar(var) if local_var: if flag in local_var: return copy.copy(local_var[flag]) return None - def delVarFlag(self,var,flag): + def delVarFlag(self, var, flag): local_var = self._findVar(var) if not local_var: return @@ -220,7 +274,7 @@ class DataSmart: if var in self.dict and flag in self.dict[var]: del self.dict[var][flag] - def setVarFlags(self,var,flags): + def setVarFlags(self, var, flags): if not var in self.dict: self._makeShadowCopy(var) @@ -229,7 +283,7 @@ class DataSmart: continue self.dict[var][i] = flags[i] - def getVarFlags(self,var): + def getVarFlags(self, var): local_var = self._findVar(var) flags = {} @@ -244,7 +298,7 @@ class DataSmart: return flags - def delVarFlags(self,var): + def delVarFlags(self, var): if not var in self.dict: self._makeShadowCopy(var) @@ -274,21 +328,19 @@ class DataSmart: def keys(self): def _keys(d, mykey): if "_data" in d: - _keys(d["_data"],mykey) + _keys(d["_data"], mykey) for key in d.keys(): if key != "_data": mykey[key] = None keytab = {} - _keys(self.dict,keytab) + _keys(self.dict, keytab) return keytab.keys() - def __getitem__(self,item): + def __getitem__(self, item): #print "Warning deprecated" return self.getVar(item, False) - def __setitem__(self,var,data): + def __setitem__(self, var, data): #print "Warning deprecated" - self.setVar(var,data) - - + self.setVar(var, data) diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py index afd5bf57c..7731649ef 100644 --- a/bitbake/lib/bb/event.py +++ b/bitbake/lib/bb/event.py @@ -22,7 +22,8 @@ BitBake build tools. # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os, re, sys +import os, sys +import warnings import bb.utils import pickle @@ -38,7 +39,7 @@ class Event: self.pid = worker_pid NotHandled = 0 -Handled = 1 +Handled = 1 Registered = 10 AlreadyRegistered = 14 @@ -48,13 +49,25 @@ _handlers = {} _ui_handlers = {} _ui_handler_seq = 0 +# For compatibility +bb.utils._context["NotHandled"] = NotHandled +bb.utils._context["Handled"] = Handled + def fire_class_handlers(event, d): + import bb.msg + if isinstance(event, bb.msg.MsgBase): + return + for handler in _handlers: h = _handlers[handler] event.data = d if type(h).__name__ == "code": - exec(h) - tmpHandler(event) + locals = {"e": event} + bb.utils.simple_exec(h, locals) + ret = bb.utils.better_eval("tmpHandler(e)", locals) + if ret is not None: + warnings.warn("Using Handled/NotHandled in event handlers is deprecated", + DeprecationWarning, stacklevel = 2) else: h(event) del event.data @@ -76,9 +89,9 @@ def fire_ui_handlers(event, d): def fire(event, d): """Fire off an Event""" - # We can fire class handlers in the worker process context and this is + # We can fire class handlers in the worker process context and this is # desired so they get the task based datastore. - # UI handlers need to be fired in the server context so we defer this. They + # UI handlers need to be fired in the server context so we defer this. They # don't have a datastore so the datastore context isn't a problem. fire_class_handlers(event, d) @@ -91,13 +104,13 @@ def worker_fire(event, d): data = "<event>" + pickle.dumps(event) + "</event>" try: if os.write(worker_pipe, data) != len (data): - print "Error sending event to server (short write)" + print("Error sending event to server (short write)") except OSError: sys.exit(1) def fire_from_worker(event, d): if not event.startswith("<event>") or not event.endswith("</event>"): - print "Error, not an event" + print("Error, not an event") return event = pickle.loads(event[7:-8]) fire_ui_handlers(event, d) @@ -222,10 +235,11 @@ class BuildCompleted(BuildBase): class NoProvider(Event): """No Provider for an Event""" - def __init__(self, item, runtime=False): + def __init__(self, item, runtime=False, dependees=None): Event.__init__(self) self._item = item self._runtime = runtime + self._dependees = dependees def getItem(self): return self._item @@ -284,4 +298,3 @@ class DepTreeGenerated(Event): def __init__(self, depgraph): Event.__init__(self) self._depgraph = depgraph - diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index b566da431..31b965379 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py @@ -24,6 +24,8 @@ BitBake build tools. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig +from __future__ import absolute_import +from __future__ import print_function import os, re import bb from bb import data @@ -53,24 +55,6 @@ class InvalidSRCREV(Exception): def decodeurl(url): """Decodes an URL into the tokens (scheme, network location, path, user, password, parameters). - - >>> decodeurl("http://www.google.com/index.html") - ('http', 'www.google.com', '/index.html', '', '', {}) - - >>> decodeurl("file://gas/COPYING") - ('file', '', 'gas/COPYING', '', '', {}) - - CVS url with username, host and cvsroot. The cvs module to check out is in the - parameters: - - >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg") - ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}) - - Dito, but this time the username has a password part. And we also request a special tag - to check out. - - >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81") - ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}) """ m = re.compile('(?P<type>[^:]*)://((?P<user>.+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) @@ -103,7 +87,7 @@ def decodeurl(url): p = {} if parm: for s in parm.split(';'): - s1,s2 = s.split('=') + s1, s2 = s.split('=') p[s1] = s2 return (type, host, path, user, pswd, p) @@ -111,27 +95,12 @@ def decodeurl(url): def encodeurl(decoded): """Encodes a URL from tokens (scheme, network location, path, user, password, parameters). - - >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}]) - 'http://www.google.com/index.html' - - CVS with username, host and cvsroot. The cvs module to check out is in the - parameters: - - >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}]) - 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg' - - Dito, but this time the username has a password part. And we also request a special tag - to check out. - - >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}]) - 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg' """ (type, host, path, user, pswd, p) = decoded if not type or not path: - bb.msg.fatal(bb.msg.domain.Fetcher, "invalid or missing parameters for url encoding") + raise MissingParameterError("Type or path url components missing when encoding %s" % decoded) url = '%s://' % type if user: url += "%s" % user @@ -151,15 +120,14 @@ def uri_replace(uri, uri_find, uri_replace, d): # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri) if not uri or not uri_find or not uri_replace: bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing") - uri_decoded = list(bb.decodeurl(uri)) - uri_find_decoded = list(bb.decodeurl(uri_find)) - uri_replace_decoded = list(bb.decodeurl(uri_replace)) - result_decoded = ['','','','','',{}] + uri_decoded = list(decodeurl(uri)) + uri_find_decoded = list(decodeurl(uri_find)) + uri_replace_decoded = list(decodeurl(uri_replace)) + result_decoded = ['', '', '', '', '', {}] for i in uri_find_decoded: loc = uri_find_decoded.index(i) result_decoded[loc] = uri_decoded[loc] - import types - if type(i) == types.StringType: + if isinstance(i, basestring): if (re.match(i, uri_decoded[loc])): result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) if uri_find_decoded.index(i) == 2: @@ -174,19 +142,20 @@ def uri_replace(uri, uri_find, uri_replace, d): # else: # for j in i: # FIXME: apply replacements against options - return bb.encodeurl(result_decoded) + return encodeurl(result_decoded) methods = [] urldata_cache = {} saved_headrevs = {} +persistent_database_connection = {} def fetcher_init(d): """ - Called to initilize the fetchers once the configuration data is known + Called to initialize the fetchers once the configuration data is known. Calls before this must not hit the cache. """ - pd = persist_data.PersistData(d) - # When to drop SCM head revisions controled by user policy + pd = persist_data.PersistData(d, persistent_database_connection) + # When to drop SCM head revisions controlled by user policy srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear" if srcrev_policy == "cache": bb.msg.debug(1, bb.msg.domain.Fetcher, "Keeping SRCREV cache due to cache policy of: %s" % srcrev_policy) @@ -198,7 +167,7 @@ def fetcher_init(d): pass pd.delDomain("BB_URI_HEADREVS") else: - bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy) + raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) for m in methods: if hasattr(m, "init"): @@ -214,7 +183,7 @@ def fetcher_compare_revisons(d): return true/false on whether they've changed. """ - pd = persist_data.PersistData(d) + pd = persist_data.PersistData(d, persistent_database_connection) data = pd.getKeyValues("BB_URI_HEADREVS") data2 = bb.fetch.saved_headrevs @@ -236,6 +205,7 @@ def fetcher_compare_revisons(d): def init(urls, d, setup = True): urldata = {} + fn = bb.data.getVar('FILE', d, 1) if fn in urldata_cache: urldata = urldata_cache[fn] @@ -247,7 +217,7 @@ def init(urls, d, setup = True): if setup: for url in urldata: if not urldata[url].setup: - urldata[url].setup_localpath(d) + urldata[url].setup_localpath(d) urldata_cache[fn] = urldata return urldata @@ -265,7 +235,7 @@ def go(d, urls = None): ud = urldata[u] m = ud.method if ud.localfile: - if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5): + if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5) and os.path.exists(ud.localfile): # File already present along with md5 stamp file # Touch md5 file to show activity try: @@ -275,8 +245,8 @@ def go(d, urls = None): pass continue lf = bb.utils.lockfile(ud.lockfile) - if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5): - # If someone else fetched this before we got the lock, + if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5) and os.path.exists(ud.localfile): + # If someone else fetched this before we got the lock, # notice and don't try again try: os.utime(ud.md5, None) @@ -332,7 +302,7 @@ def checkstatus(d): ret = try_mirrors (d, u, mirrors, True) if not ret: - bb.msg.error(bb.msg.domain.Fetcher, "URL %s doesn't work" % u) + raise FetchError("URL %s doesn't work" % u) def localpaths(d): """ @@ -342,7 +312,7 @@ def localpaths(d): urldata = init([], d, True) for u in urldata: - ud = urldata[u] + ud = urldata[u] local.append(ud.localpath) return local @@ -354,15 +324,15 @@ def get_srcrev(d): Return the version string for the current package (usually to be used as PV) Most packages usually only have one SCM so we just pass on the call. - In the multi SCM case, we build a value based on SRCREV_FORMAT which must + In the multi SCM case, we build a value based on SRCREV_FORMAT which must have been set. """ # - # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which + # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which # could translate into a call to here. If it does, we need to catch this # and provide some way so it knows get_srcrev is active instead of being - # some number etc. hence the srcrev_internal_call tracking and the magic + # some number etc. hence the srcrev_internal_call tracking and the magic # "SRCREVINACTION" return value. # # Neater solutions welcome! @@ -372,7 +342,7 @@ def get_srcrev(d): scms = [] - # Only call setup_localpath on URIs which suppports_srcrev() + # Only call setup_localpath on URIs which suppports_srcrev() urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d, False) for u in urldata: ud = urldata[u] @@ -385,7 +355,7 @@ def get_srcrev(d): bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI") raise ParameterError - bb.data.setVar('__BB_DONT_CACHE','1', d) + bb.data.setVar('__BB_DONT_CACHE', '1', d) if len(scms) == 1: return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d) @@ -408,7 +378,7 @@ def get_srcrev(d): def localpath(url, d, cache = True): """ - Called from the parser with cache=False since the cache isn't ready + Called from the parser with cache=False since the cache isn't ready at this point. Also called from classed in OE e.g. patch.bbclass """ ud = init([url], d) @@ -432,7 +402,7 @@ def runfetchcmd(cmd, d, quiet = False): for var in exportvars: val = data.getVar(var, d, True) if val: - cmd = 'export ' + var + '=%s; %s' % (val, cmd) + cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd) @@ -440,12 +410,12 @@ def runfetchcmd(cmd, d, quiet = False): stdout_handle = os.popen(cmd + " 2>&1", "r") output = "" - while 1: + while True: line = stdout_handle.readline() if not line: break if not quiet: - print line, + print(line, end=' ') output += line status = stdout_handle.close() or 0 @@ -507,7 +477,7 @@ class FetchData(object): """ def __init__(self, url, d): self.localfile = "" - (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d)) + (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d)) self.date = Fetch.getSRCDate(self, d) self.url = url if not self.user and "user" in self.parm: @@ -571,7 +541,7 @@ class Fetch(object): def localpath(self, url, urldata, d): """ Return the local filename of a given url assuming a successful fetch. - Can also setup variables in urldata for use in go (saving code duplication + Can also setup variables in urldata for use in go (saving code duplication and duplicate code execution) """ return url @@ -632,8 +602,8 @@ class Fetch(object): """ Return: a) a source revision if specified - b) True if auto srcrev is in action - c) False otherwise + b) True if auto srcrev is in action + c) False otherwise """ if 'rev' in ud.parm: @@ -665,7 +635,7 @@ class Fetch(object): b) None otherwise """ - localcount= None + localcount = None if 'name' in ud.parm: pn = data.getVar("PN", d, 1) localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1) @@ -706,7 +676,7 @@ class Fetch(object): if not hasattr(self, "_latest_revision"): raise ParameterError - pd = persist_data.PersistData(d) + pd = persist_data.PersistData(d, persistent_database_connection) key = self.generate_revision_key(url, ud, d) rev = pd.getValue("BB_URI_HEADREVS", key) if rev != None: @@ -718,12 +688,12 @@ class Fetch(object): def sortable_revision(self, url, ud, d): """ - + """ if hasattr(self, "_sortable_revision"): return self._sortable_revision(url, ud, d) - pd = persist_data.PersistData(d) + pd = persist_data.PersistData(d, persistent_database_connection) key = self.generate_revision_key(url, ud, d) latest_rev = self._build_revision(url, ud, d) @@ -758,18 +728,18 @@ class Fetch(object): key = self._revision_key(url, ud, d) return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "") -import cvs -import git -import local -import svn -import wget -import svk -import ssh -import perforce -import bzr -import hg -import osc -import repo +from . import cvs +from . import git +from . import local +from . import svn +from . import wget +from . import svk +from . import ssh +from . import perforce +from . import bzr +from . import hg +from . import osc +from . import repo methods.append(local.Local()) methods.append(wget.Wget()) diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py index c6e33c334..813d7d8c8 100644 --- a/bitbake/lib/bb/fetch/bzr.py +++ b/bitbake/lib/bb/fetch/bzr.py @@ -46,15 +46,15 @@ class Bzr(Fetch): revision = Fetch.srcrev_internal_helper(ud, d) if revision is True: - ud.revision = self.latest_revision(url, ud, d) + ud.revision = self.latest_revision(url, ud, d) elif revision: ud.revision = revision if not ud.revision: - ud.revision = self.latest_revision(url, ud, d) + ud.revision = self.latest_revision(url, ud, d) ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) - + return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) def _buildbzrcommand(self, ud, d, command): @@ -145,4 +145,3 @@ class Bzr(Fetch): def _build_revision(self, url, ud, d): return ud.revision - diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py index 443f52131..61976f7ef 100644 --- a/bitbake/lib/bb/fetch/cvs.py +++ b/bitbake/lib/bb/fetch/cvs.py @@ -139,8 +139,8 @@ class Cvs(Fetch): bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory") pkg = data.expand('${PN}', d) pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) - moddir = os.path.join(pkgdir,localdir) - if os.access(os.path.join(moddir,'CVS'), os.R_OK): + moddir = os.path.join(pkgdir, localdir) + if os.access(os.path.join(moddir, 'CVS'), os.R_OK): bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(moddir) @@ -157,7 +157,7 @@ class Cvs(Fetch): try: os.rmdir(moddir) except OSError: - pass + pass raise FetchError(ud.module) # tar them up to a defined filename diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py index 41ebc5b99..8c91de9db 100644 --- a/bitbake/lib/bb/fetch/git.py +++ b/bitbake/lib/bb/fetch/git.py @@ -57,12 +57,12 @@ class Git(Fetch): tag = Fetch.srcrev_internal_helper(ud, d) if tag is True: - ud.tag = self.latest_revision(url, ud, d) + ud.tag = self.latest_revision(url, ud, d) elif tag: ud.tag = tag if not ud.tag or ud.tag == "master": - ud.tag = self.latest_revision(url, ud, d) + ud.tag = self.latest_revision(url, ud, d) subdir = ud.parm.get("subpath", "") if subdir != "": @@ -114,7 +114,7 @@ class Git(Fetch): os.chdir(ud.clonedir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) - if mirror_tarballs != "0" or 'fullclone' in ud.parm: + if mirror_tarballs != "0" or 'fullclone' in ud.parm: bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) @@ -188,7 +188,7 @@ class Git(Fetch): def _sortable_buildindex_disabled(self, url, ud, d, rev): """ - Return a suitable buildindex for the revision specified. This is done by counting revisions + Return a suitable buildindex for the revision specified. This is done by counting revisions using "git rev-list" which may or may not work in different circumstances. """ @@ -197,7 +197,7 @@ class Git(Fetch): # Check if we have the rev already if not os.path.exists(ud.clonedir): - print "no repo" + print("no repo") self.go(None, ud, d) if not os.path.exists(ud.clonedir): bb.msg.error(bb.msg.domain.Fetcher, "GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value" % (url, ud.clonedir)) @@ -213,5 +213,4 @@ class Git(Fetch): buildindex = "%s" % output.split()[0] bb.msg.debug(1, bb.msg.domain.Fetcher, "GIT repository for %s in %s is returning %s revisions in rev-list before %s" % (url, ud.clonedir, buildindex, rev)) - return buildindex - + return buildindex diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py index d0756382f..efb3b5c76 100644 --- a/bitbake/lib/bb/fetch/hg.py +++ b/bitbake/lib/bb/fetch/hg.py @@ -134,9 +134,9 @@ class Hg(Fetch): os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd) runfetchcmd(fetchcmd, d) - - # Even when we clone (fetch), we still need to update as hg's clone - # won't checkout the specified revision if its on a branch + + # Even when we clone (fetch), we still need to update as hg's clone + # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd) runfetchcmd(updatecmd, d) @@ -170,4 +170,3 @@ class Hg(Fetch): Return a unique key for the url """ return "hg:" + ud.moddir - diff --git a/bitbake/lib/bb/fetch/local.py b/bitbake/lib/bb/fetch/local.py index f9bdf589c..882a2c460 100644 --- a/bitbake/lib/bb/fetch/local.py +++ b/bitbake/lib/bb/fetch/local.py @@ -27,6 +27,7 @@ BitBake build tools. import os import bb +import bb.utils from bb import data from bb.fetch import Fetch @@ -47,7 +48,7 @@ class Local(Fetch): if path[0] != "/": filespath = data.getVar('FILESPATH', d, 1) if filespath: - newpath = bb.which(filespath, path) + newpath = bb.utils.which(filespath, path) if not newpath: filesdir = data.getVar('FILESDIR', d, 1) if filesdir: @@ -65,8 +66,8 @@ class Local(Fetch): Check the status of the url """ if urldata.localpath.find("*") != -1: - bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s looks like a glob and was therefore not checked." % url) - return True + bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s looks like a glob and was therefore not checked." % url) + return True if os.path.exists(urldata.localpath): - return True + return True return False diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py index 548dd9d07..ed773939b 100644 --- a/bitbake/lib/bb/fetch/osc.py +++ b/bitbake/lib/bb/fetch/osc.py @@ -16,7 +16,7 @@ from bb.fetch import MissingParameterError from bb.fetch import runfetchcmd class Osc(Fetch): - """Class to fetch a module or modules from Opensuse build server + """Class to fetch a module or modules from Opensuse build server repositories.""" def supports(self, url, ud, d): @@ -64,7 +64,7 @@ class Osc(Fetch): proto = "ocs" if "proto" in ud.parm: proto = ud.parm["proto"] - + options = [] config = "-c %s" % self.generate_config(ud, d) @@ -108,7 +108,7 @@ class Osc(Fetch): os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd) runfetchcmd(oscfetchcmd, d) - + os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename try: @@ -131,7 +131,7 @@ class Osc(Fetch): config_path = "%s/oscrc" % data.expand('${OSCDIR}', d) if (os.path.exists(config_path)): - os.remove(config_path) + os.remove(config_path) f = open(config_path, 'w') f.write("[general]\n") @@ -146,5 +146,5 @@ class Osc(Fetch): f.write("user = %s\n" % ud.parm["user"]) f.write("pass = %s\n" % ud.parm["pswd"]) f.close() - + return config_path diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py index e2c342108..1c74cff34 100644 --- a/bitbake/lib/bb/fetch/perforce.py +++ b/bitbake/lib/bb/fetch/perforce.py @@ -25,6 +25,7 @@ BitBake build tools. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig +from future_builtins import zip import os import bb from bb import data @@ -35,15 +36,15 @@ class Perforce(Fetch): def supports(self, url, ud, d): return ud.type in ['p4'] - def doparse(url,d): + def doparse(url, d): parm = {} path = url.split("://")[1] delim = path.find("@"); if delim != -1: - (user,pswd,host,port) = path.split('@')[0].split(":") + (user, pswd, host, port) = path.split('@')[0].split(":") path = path.split('@')[1] else: - (host,port) = data.getVar('P4PORT', d).split(':') + (host, port) = data.getVar('P4PORT', d).split(':') user = "" pswd = "" @@ -53,19 +54,19 @@ class Perforce(Fetch): plist = path.split(';') for item in plist: if item.count('='): - (key,value) = item.split('=') + (key, value) = item.split('=') keys.append(key) values.append(value) - parm = dict(zip(keys,values)) + parm = dict(zip(keys, values)) path = "//" + path.split(';')[0] host += ":%s" % (port) parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm) - return host,path,user,pswd,parm + return host, path, user, pswd, parm doparse = staticmethod(doparse) - def getcset(d, depot,host,user,pswd,parm): + def getcset(d, depot, host, user, pswd, parm): p4opt = "" if "cset" in parm: return parm["cset"]; @@ -95,9 +96,9 @@ class Perforce(Fetch): return cset.split(' ')[1] getcset = staticmethod(getcset) - def localpath(self, url, ud, d): + def localpath(self, url, ud, d): - (host,path,user,pswd,parm) = Perforce.doparse(url,d) + (host, path, user, pswd, parm) = Perforce.doparse(url, d) # If a label is specified, we use that as our filename @@ -115,7 +116,7 @@ class Perforce(Fetch): cset = Perforce.getcset(d, path, host, user, pswd, parm) - ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host,base.replace('/', '.'), cset), d) + ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d) return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile) @@ -124,7 +125,7 @@ class Perforce(Fetch): Fetch urls """ - (host,depot,user,pswd,parm) = Perforce.doparse(loc, d) + (host, depot, user, pswd, parm) = Perforce.doparse(loc, d) if depot.find('/...') != -1: path = depot[:depot.find('/...')] @@ -160,14 +161,14 @@ class Perforce(Fetch): tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: - bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") + bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") raise FetchError(module) if "label" in parm: - depot = "%s@%s" % (depot,parm["label"]) + depot = "%s@%s" % (depot, parm["label"]) else: cset = Perforce.getcset(d, depot, host, user, pswd, parm) - depot = "%s@%s" % (depot,cset) + depot = "%s@%s" % (depot, cset) os.chdir(tmpfile) bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) @@ -175,12 +176,12 @@ class Perforce(Fetch): p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot)) if not p4file: - bb.error("Fetch: unable to get the P4 files from %s" % (depot)) + bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to get the P4 files from %s" % (depot)) raise FetchError(module) count = 0 - for file in p4file: + for file in p4file: list = file.split() if list[2] == "delete": @@ -189,11 +190,11 @@ class Perforce(Fetch): dest = list[0][len(path)+1:] where = dest.find("#") - os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module,dest[:where],list[0])) + os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0])) count = count + 1 - + if count == 0: - bb.error("Fetch: No files gathered from the P4 fetch") + bb.msg.error(bb.msg.domain.Fetcher, "Fetch: No files gathered from the P4 fetch") raise FetchError(module) myret = os.system("tar -czf %s %s" % (ud.localpath, module)) @@ -205,5 +206,3 @@ class Perforce(Fetch): raise FetchError(module) # cleanup os.system('rm -rf %s' % tmpfile) - - diff --git a/bitbake/lib/bb/fetch/repo.py b/bitbake/lib/bb/fetch/repo.py index 34c32fe0b..883310b01 100644 --- a/bitbake/lib/bb/fetch/repo.py +++ b/bitbake/lib/bb/fetch/repo.py @@ -23,11 +23,10 @@ BitBake "Fetch" repo (git) implementation # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os, re +import os import bb from bb import data from bb.fetch import Fetch -from bb.fetch import FetchError from bb.fetch import runfetchcmd class Repo(Fetch): diff --git a/bitbake/lib/bb/fetch/ssh.py b/bitbake/lib/bb/fetch/ssh.py index 68e6fdb1d..86c76f4e4 100644 --- a/bitbake/lib/bb/fetch/ssh.py +++ b/bitbake/lib/bb/fetch/ssh.py @@ -114,5 +114,5 @@ class SSH(Fetch): (exitstatus, output) = commands.getstatusoutput(cmd) if exitstatus != 0: - print output + print(output) raise FetchError('Unable to fetch %s' % url) diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py index ba9f6ab10..375e8df05 100644 --- a/bitbake/lib/bb/fetch/svn.py +++ b/bitbake/lib/bb/fetch/svn.py @@ -78,7 +78,7 @@ class Svn(Fetch): ud.revision = rev ud.date = "" else: - ud.revision = "" + ud.revision = "" ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py index ae1c6ad13..dcc58c75e 100644 --- a/bitbake/lib/bb/fetch/wget.py +++ b/bitbake/lib/bb/fetch/wget.py @@ -30,6 +30,8 @@ import bb from bb import data from bb.fetch import Fetch from bb.fetch import FetchError +from bb.fetch import encodeurl, decodeurl +from bb.fetch import runfetchcmd class Wget(Fetch): """Class to fetch urls via 'wget'""" @@ -37,11 +39,11 @@ class Wget(Fetch): """ Check to see if a given url can be fetched with wget. """ - return ud.type in ['http','https','ftp'] + return ud.type in ['http', 'https', 'ftp'] def localpath(self, url, ud, d): - url = bb.encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}]) + url = encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}]) ud.basename = os.path.basename(ud.path) ud.localfile = data.expand(os.path.basename(url), d) @@ -60,37 +62,16 @@ class Wget(Fetch): fetchcmd = data.getVar("FETCHCOMMAND", d, 1) uri = uri.split(";")[0] - uri_decoded = list(bb.decodeurl(uri)) + uri_decoded = list(decodeurl(uri)) uri_type = uri_decoded[0] uri_host = uri_decoded[1] - bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri) fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0]) fetchcmd = fetchcmd.replace("${FILE}", ud.basename) - httpproxy = None - ftpproxy = None - if uri_type == 'http': - httpproxy = data.getVar("HTTP_PROXY", d, True) - httpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split() - for p in httpproxy_ignore: - if uri_host.endswith(p): - httpproxy = None - break - if uri_type == 'ftp': - ftpproxy = data.getVar("FTP_PROXY", d, True) - ftpproxy_ignore = (data.getVar("HTTP_PROXY_IGNORE", d, True) or "").split() - for p in ftpproxy_ignore: - if uri_host.endswith(p): - ftpproxy = None - break - if httpproxy: - fetchcmd = "http_proxy=" + httpproxy + " " + fetchcmd - if ftpproxy: - fetchcmd = "ftp_proxy=" + ftpproxy + " " + fetchcmd + + bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri) bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd) - ret = os.system(fetchcmd) - if ret != 0: - return False + runfetchcmd(fetchcmd, d) # Sanity check since wget can pretend it succeed when it didn't # Also, this used to happen if sourceforge sent us to the mirror page diff --git a/bitbake/lib/bb/methodpool.py b/bitbake/lib/bb/methodpool.py index f43c4a058..1485b1357 100644 --- a/bitbake/lib/bb/methodpool.py +++ b/bitbake/lib/bb/methodpool.py @@ -27,7 +27,7 @@ a method pool to do this task. This pool will be used to compile and execute the functions. It - will be smart enough to + will be smart enough to """ from bb.utils import better_compile, better_exec @@ -43,8 +43,8 @@ def insert_method(modulename, code, fn): Add code of a module should be added. The methods will be simply added, no checking will be done """ - comp = better_compile(code, "<bb>", fn ) - better_exec(comp, __builtins__, code, fn) + comp = better_compile(code, modulename, fn ) + better_exec(comp, None, code, fn) # now some instrumentation code = comp.co_names @@ -59,7 +59,7 @@ def insert_method(modulename, code, fn): def check_insert_method(modulename, code, fn): """ Add the code if it wasnt added before. The module - name will be used for that + name will be used for that Variables: @modulename a short name e.g. base.bbclass @@ -81,4 +81,4 @@ def get_parsed_dict(): """ shortcut """ - return _parsed_methods + return _parsed_methods diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py index 3fcf7091b..cea5efb5a 100644 --- a/bitbake/lib/bb/msg.py +++ b/bitbake/lib/bb/msg.py @@ -22,26 +22,32 @@ Message handling infrastructure for bitbake # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import sys, bb -from bb import event - -debug_level = {} +import sys +import collections +import bb +import bb.event +debug_level = collections.defaultdict(lambda: 0) verbose = False -domain = bb.utils.Enum( - 'Build', - 'Cache', - 'Collection', - 'Data', - 'Depends', - 'Fetcher', - 'Parsing', - 'PersistData', - 'Provider', - 'RunQueue', - 'TaskData', - 'Util') +def _NamedTuple(name, fields): + Tuple = collections.namedtuple(name, " ".join(fields)) + return Tuple(*range(len(fields))) + +domain = _NamedTuple("Domain", ( + "Default", + "Build", + "Cache", + "Collection", + "Data", + "Depends", + "Fetcher", + "Parsing", + "PersistData", + "Provider", + "RunQueue", + "TaskData", + "Util")) class MsgBase(bb.event.Event): @@ -49,7 +55,7 @@ class MsgBase(bb.event.Event): def __init__(self, msg): self._message = msg - event.Event.__init__(self) + bb.event.Event.__init__(self) class MsgDebug(MsgBase): """Debug Message""" @@ -74,52 +80,62 @@ class MsgPlain(MsgBase): # def set_debug_level(level): - bb.msg.debug_level = {} - for domain in bb.msg.domain: - bb.msg.debug_level[domain] = level - bb.msg.debug_level['default'] = level + for d in domain: + debug_level[d] = level + debug_level[domain.Default] = level + +def get_debug_level(msgdomain = domain.Default): + return debug_level[msgdomain] def set_verbose(level): - bb.msg.verbose = level - -def set_debug_domains(domains): - for domain in domains: - found = False - for ddomain in bb.msg.domain: - if domain == str(ddomain): - bb.msg.debug_level[ddomain] = bb.msg.debug_level[ddomain] + 1 - found = True - if not found: - bb.msg.warn(None, "Logging domain %s is not valid, ignoring" % domain) + verbose = level + +def set_debug_domains(strdomains): + for domainstr in strdomains: + for d in domain: + if domain._fields[d] == domainstr: + debug_level[d] += 1 + break + else: + warn(None, "Logging domain %s is not valid, ignoring" % domainstr) # # Message handling functions # -def debug(level, domain, msg, fn = None): - if not domain: - domain = 'default' - if debug_level[domain] >= level: +def debug(level, msgdomain, msg, fn = None): + if not msgdomain: + msgdomain = domain.Default + + if debug_level[msgdomain] >= level: bb.event.fire(MsgDebug(msg), None) + if not bb.event._ui_handlers: + print('DEBUG: ' + msg) + +def note(level, msgdomain, msg, fn = None): + if not msgdomain: + msgdomain = domain.Default -def note(level, domain, msg, fn = None): - if not domain: - domain = 'default' - if level == 1 or verbose or debug_level[domain] >= 1: + if level == 1 or verbose or debug_level[msgdomain] >= 1: bb.event.fire(MsgNote(msg), None) + if not bb.event._ui_handlers: + print('NOTE: ' + msg) -def warn(domain, msg, fn = None): +def warn(msgdomain, msg, fn = None): bb.event.fire(MsgWarn(msg), None) + if not bb.event._ui_handlers: + print('WARNING: ' + msg) -def error(domain, msg, fn = None): +def error(msgdomain, msg, fn = None): bb.event.fire(MsgError(msg), None) print 'ERROR: ' + msg -def fatal(domain, msg, fn = None): +def fatal(msgdomain, msg, fn = None): bb.event.fire(MsgFatal(msg), None) - print 'FATAL: ' + msg + print('FATAL: ' + msg) sys.exit(1) def plain(msg, fn = None): bb.event.fire(MsgPlain(msg), None) - + if not bb.event._ui_handlers: + print(msg) diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py index 2a7897cdf..95f372b00 100644 --- a/bitbake/lib/bb/parse/__init__.py +++ b/bitbake/lib/bb/parse/__init__.py @@ -24,11 +24,10 @@ File parsers for the BitBake build tools. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig -__all__ = [ 'ParseError', 'SkipPackage', 'cached_mtime', 'mark_dependency', - 'supports', 'handle', 'init' ] handlers = [] import bb, os +import bb.utils class ParseError(Exception): """Exception raised when parsing fails""" @@ -38,12 +37,12 @@ class SkipPackage(Exception): __mtime_cache = {} def cached_mtime(f): - if not __mtime_cache.has_key(f): + if f not in __mtime_cache: __mtime_cache[f] = os.stat(f)[8] return __mtime_cache[f] def cached_mtime_noerror(f): - if not __mtime_cache.has_key(f): + if f not in __mtime_cache: try: __mtime_cache[f] = os.stat(f)[8] except OSError: @@ -57,8 +56,8 @@ def update_mtime(f): def mark_dependency(d, f): if f.startswith('./'): f = "%s/%s" % (os.getcwd(), f[2:]) - deps = bb.data.getVar('__depends', d) or [] - deps.append( (f, cached_mtime(f)) ) + deps = bb.data.getVar('__depends', d) or set() + deps.update([(f, cached_mtime(f))]) bb.data.setVar('__depends', deps, d) def supports(fn, data): @@ -82,9 +81,11 @@ def init(fn, data): def resolve_file(fn, d): if not os.path.isabs(fn): - fn = bb.which(bb.data.getVar("BBPATH", d, 1), fn) - if not fn: - raise IOError("file %s not found" % fn) + bbpath = bb.data.getVar("BBPATH", d, True) + newfn = bb.which(bbpath, fn) + if not newfn: + raise IOError("file %s not found in %s" % (fn, bbpath)) + fn = newfn bb.msg.debug(2, bb.msg.domain.Parsing, "LOAD %s" % fn) return fn diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py index 59aa44bee..dae2e1115 100644 --- a/bitbake/lib/bb/parse/ast.py +++ b/bitbake/lib/bb/parse/ast.py @@ -21,8 +21,11 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +from __future__ import absolute_import +from future_builtins import filter import bb, re, string -from itertools import chain +from bb import methodpool +import itertools __word__ = re.compile(r"\S+") __parsed_methods__ = bb.methodpool.get_parsed_dict() @@ -30,7 +33,8 @@ _bbversions_re = re.compile(r"\[(?P<from>[0-9]+)-(?P<to>[0-9]+)\]") class StatementGroup(list): def eval(self, data): - map(lambda x: x.eval(data), self) + for statement in self: + statement.eval(data) class AstNode(object): pass @@ -103,7 +107,6 @@ class DataNode(AstNode): val = groupd["value"] if 'flag' in groupd and groupd['flag'] != None: - bb.msg.debug(3, bb.msg.domain.Parsing, "setVarFlag(%s, %s, %s, data)" % (key, groupd['flag'], val)) bb.data.setVarFlag(key, groupd['flag'], val, data) elif groupd["lazyques"]: assigned = bb.data.getVar("__lazy_assigned", data) or [] @@ -143,7 +146,7 @@ class PythonMethodNode(AstNode): # Note we will add root to parsedmethods after having parse # 'this' file. This means we will not parse methods from # bb classes twice - if not self.root in __parsed_methods__: + if not bb.methodpool.parsed_module(self.root): text = '\n'.join(self.body) bb.methodpool.insert_method(self.root, text, self.fn) @@ -254,7 +257,7 @@ class InheritNode(AstNode): def eval(self, data): bb.parse.BBHandler.inherit(self.n, data) - + def handleInclude(statements, m, fn, lineno, force): statements.append(IncludeNode(m.group(1), fn, lineno, force)) @@ -293,7 +296,7 @@ def handleInherit(statements, m): n = __word__.findall(files) statements.append(InheritNode(m.group(1))) -def finalise(fn, d): +def finalize(fn, d): for lazykey in bb.data.getVar("__lazy_assigned", d) or (): if bb.data.getVar(lazykey, d) is None: val = bb.data.getVarFlag(lazykey, "defaultval", d) @@ -301,35 +304,16 @@ def finalise(fn, d): bb.data.expandKeys(d) bb.data.update_data(d) - anonqueue = bb.data.getVar("__anonqueue", d, 1) or [] - body = [x['content'] for x in anonqueue] - flag = { 'python' : 1, 'func' : 1 } - bb.data.setVar("__anonfunc", "\n".join(body), d) - bb.data.setVarFlags("__anonfunc", flag, d) - from bb import build - try: - t = bb.data.getVar('T', d) - bb.data.setVar('T', '${TMPDIR}/anonfunc/', d) - anonfuncs = bb.data.getVar('__BBANONFUNCS', d) or [] - code = "" - for f in anonfuncs: - code = code + " %s(d)\n" % f - bb.data.setVar("__anonfunc", code, d) - build.exec_func("__anonfunc", d) - bb.data.delVar('T', d) - if t: - bb.data.setVar('T', t, d) - except Exception, e: - bb.msg.debug(1, bb.msg.domain.Parsing, "Exception when executing anonymous function: %s" % e) - raise - bb.data.delVar("__anonqueue", d) - bb.data.delVar("__anonfunc", d) + code = [] + for funcname in bb.data.getVar("__BBANONFUNCS", d) or []: + code.append("%s(d)" % funcname) + bb.utils.simple_exec("\n".join(code), {"d": d}) bb.data.update_data(d) - all_handlers = {} + all_handlers = {} for var in bb.data.getVar('__BBHANDLERS', d) or []: # try to add the handler - handler = bb.data.getVar(var,d) + handler = bb.data.getVar(var, d) bb.event.register(var, handler) tasklist = bb.data.getVar('__BBTASKS', d) or [] @@ -360,7 +344,7 @@ def _expand_versions(versions): versions = iter(versions) while True: try: - version = versions.next() + version = next(versions) except StopIteration: break @@ -370,14 +354,14 @@ def _expand_versions(versions): else: newversions = expand_one(version, int(range_ver.group("from")), int(range_ver.group("to"))) - versions = chain(newversions, versions) + versions = itertools.chain(newversions, versions) def multi_finalize(fn, d): safe_d = d d = bb.data.createCopy(safe_d) try: - finalise(fn, d) + finalize(fn, d) except bb.parse.SkipPackage: bb.data.setVar("__SKIPPED", True, d) datastores = {"": safe_d} @@ -420,7 +404,7 @@ def multi_finalize(fn, d): d = bb.data.createCopy(safe_d) verfunc(pv, d, safe_d) try: - finalise(fn, d) + finalize(fn, d) except bb.parse.SkipPackage: bb.data.setVar("__SKIPPED", True, d) @@ -436,15 +420,15 @@ def multi_finalize(fn, d): safe_d.setVar("BBCLASSEXTEND", extended) _create_variants(datastores, extended.split(), extendfunc) - for variant, variant_d in datastores.items(): + for variant, variant_d in datastores.iteritems(): if variant: try: - finalise(fn, variant_d) + finalize(fn, variant_d) except bb.parse.SkipPackage: bb.data.setVar("__SKIPPED", True, variant_d) if len(datastores) > 1: - variants = filter(None, datastores.keys()) + variants = filter(None, datastores.iterkeys()) safe_d.setVar("__VARIANTS", " ".join(variants)) datastores[""] = d diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py index 262c883c9..bb5617488 100644 --- a/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -11,7 +11,7 @@ # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2003, 2004 Phil Blundell -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -25,15 +25,17 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import re, bb, os, sys, time, string +from __future__ import absolute_import +import re, bb, os import bb.fetch, bb.build, bb.utils -from bb import data, fetch +from bb import data -from ConfHandler import include, init -from bb.parse import ParseError, resolve_file, ast +from . import ConfHandler +from .. import resolve_file, ast +from .ConfHandler import include, init # For compatibility -from bb.parse import vars_from_file +bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) __func_start_regexp__ = re.compile( r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile( r"inherit\s+(.+)" ) @@ -68,8 +70,8 @@ def inherit(files, d): __inherit_cache = data.getVar('__inherit_cache', d) or [] fn = "" lineno = 0 - files = data.expand(files, d) for file in files: + file = data.expand(file, d) if file[0] != "/" and file[-8:] != ".bbclass": file = os.path.join('classes', '%s.bbclass' % file) @@ -80,17 +82,17 @@ def inherit(files, d): include(fn, file, d, "inherit") __inherit_cache = data.getVar('__inherit_cache', d) or [] -def get_statements(filename, absolsute_filename, base_name): +def get_statements(filename, absolute_filename, base_name): global cached_statements try: - return cached_statements[absolsute_filename] + return cached_statements[absolute_filename] except KeyError: - file = open(absolsute_filename, 'r') + file = open(absolute_filename, 'r') statements = ast.StatementGroup() lineno = 0 - while 1: + while True: lineno = lineno + 1 s = file.readline() if not s: break @@ -101,7 +103,7 @@ def get_statements(filename, absolsute_filename, base_name): feeder(IN_PYTHON_EOF, "", filename, base_name, statements) if filename.endswith(".bbclass") or filename.endswith(".inc"): - cached_statements[absolsute_filename] = statements + cached_statements[absolute_filename] = statements return statements def handle(fn, d, include): @@ -118,7 +120,7 @@ def handle(fn, d, include): bb.msg.debug(2, bb.msg.domain.Parsing, "BB " + fn + ": handle(data, include)") (root, ext) = os.path.splitext(os.path.basename(fn)) - base_name = "%s%s" % (root,ext) + base_name = "%s%s" % (root, ext) init(d) if ext == ".bbclass": @@ -164,7 +166,7 @@ def handle(fn, d, include): return d def feeder(lineno, s, fn, root, statements): - global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__,__infunc__, __body__, classes, bb, __residue__ + global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, classes, bb, __residue__ if __infunc__: if s == '}': __body__.append('') @@ -231,10 +233,9 @@ def feeder(lineno, s, fn, root, statements): ast.handleInherit(statements, m) return - from bb.parse import ConfHandler return ConfHandler.feeder(lineno, s, fn, statements) # Add us to the handlers list -from bb.parse import handlers +from .. import handlers handlers.append({'supports': supports, 'handle': handle, 'init': init}) del handlers diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py index f4f85de24..9128a2ef8 100644 --- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -10,7 +10,7 @@ # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2003, 2004 Phil Blundell -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -24,7 +24,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import re, bb.data, os, sys +import re, bb.data, os +import bb.utils from bb.parse import ParseError, resolve_file, ast #__config_regexp__ = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}]+)\s*(?P<colon>:)?(?P<ques>\?)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$") @@ -36,10 +37,7 @@ __export_regexp__ = re.compile( r"export\s+(.+)" ) def init(data): topdir = bb.data.getVar('TOPDIR', data) if not topdir: - topdir = os.getcwd() - bb.data.setVar('TOPDIR', topdir, data) - if not bb.data.getVar('BBPATH', data): - bb.fatal("The BBPATH environment variable must be set") + bb.data.setVar('TOPDIR', os.getcwd(), data) def supports(fn, d): @@ -60,7 +58,7 @@ def include(oldfn, fn, data, error_out): if not os.path.isabs(fn): dname = os.path.dirname(oldfn) bbpath = "%s:%s" % (dname, bb.data.getVar("BBPATH", data, 1)) - abs_fn = bb.which(bbpath, fn) + abs_fn = bb.utils.which(bbpath, fn) if abs_fn: fn = abs_fn @@ -88,7 +86,7 @@ def handle(fn, data, include): statements = ast.StatementGroup() lineno = 0 - while 1: + while True: lineno = lineno + 1 s = f.readline() if not s: break diff --git a/bitbake/lib/bb/parse/parse_py/__init__.py b/bitbake/lib/bb/parse/parse_py/__init__.py index 9e0e00add..3e658d0de 100644 --- a/bitbake/lib/bb/parse/parse_py/__init__.py +++ b/bitbake/lib/bb/parse/parse_py/__init__.py @@ -25,9 +25,9 @@ File parsers for the BitBake build tools. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig -__version__ = '1.0' -__all__ = [ 'ConfHandler', 'BBHandler'] +from __future__ import absolute_import +from . import ConfHandler +from . import BBHandler -import ConfHandler -import BBHandler +__version__ = '1.0' diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py index bc4045fe8..df0409cd8 100644 --- a/bitbake/lib/bb/persist_data.py +++ b/bitbake/lib/bb/persist_data.py @@ -16,6 +16,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import bb, os +import bb.utils try: import sqlite3 @@ -33,58 +34,63 @@ class PersistData: """ BitBake Persistent Data Store - Used to store data in a central location such that other threads/tasks can + Used to store data in a central location such that other threads/tasks can access them at some future date. - The "domain" is used as a key to isolate each data pool and in this - implementation corresponds to an SQL table. The SQL table consists of a + The "domain" is used as a key to isolate each data pool and in this + implementation corresponds to an SQL table. The SQL table consists of a simple key and value pair. Why sqlite? It handles all the locking issues for us. """ - def __init__(self, d): + def __init__(self, d, persistent_database_connection): + if "connection" in persistent_database_connection: + self.cursor = persistent_database_connection["connection"].cursor() + return self.cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True) if self.cachedir in [None, '']: bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.") try: os.stat(self.cachedir) except OSError: - bb.mkdirhier(self.cachedir) + bb.utils.mkdirhier(self.cachedir) - self.cachefile = os.path.join(self.cachedir,"bb_persist_data.sqlite3") + self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3") bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile) - self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None) + connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None) + persistent_database_connection["connection"] = connection + self.cursor = persistent_database_connection["connection"].cursor() def addDomain(self, domain): """ Should be called before any domain is used Creates it if it doesn't exist. """ - self.connection.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain) + self.cursor.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain) def delDomain(self, domain): """ Removes a domain and all the data it contains """ - self.connection.execute("DROP TABLE IF EXISTS %s;" % domain) + self.cursor.execute("DROP TABLE IF EXISTS %s;" % domain) def getKeyValues(self, domain): """ Return a list of key + value pairs for a domain """ ret = {} - data = self.connection.execute("SELECT key, value from %s;" % domain) + data = self.cursor.execute("SELECT key, value from %s;" % domain) for row in data: ret[str(row[0])] = str(row[1]) - return ret + return ret def getValue(self, domain, key): """ Return the value of a key for a domain """ - data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key]) + data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key]) for row in data: return row[1] @@ -92,7 +98,7 @@ class PersistData: """ Sets the value of a key for a domain """ - data = self.connection.execute("SELECT * from %s where key=?;" % domain, [key]) + data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key]) rows = 0 for row in data: rows = rows + 1 @@ -108,14 +114,11 @@ class PersistData: self._execute("DELETE from %s where key=?;" % domain, [key]) def _execute(self, *query): - while True: + while True: try: - self.connection.execute(*query) + self.cursor.execute(*query) return - except sqlite3.OperationalError, e: + except sqlite3.OperationalError as e: if 'database is locked' in str(e): continue raise - - - diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py index 058996ba5..58326f039 100644 --- a/bitbake/lib/bb/providers.py +++ b/bitbake/lib/bb/providers.py @@ -62,7 +62,7 @@ def sortPriorities(pn, dataCache, pkg_pn = None): def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): """ Check if the version pe,pv,pr is the preferred one. - If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%' + If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%' """ if (pr == preferred_r or preferred_r == None): if (pe == preferred_e or preferred_e == None): @@ -103,7 +103,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): for file_set in pkg_pn: for f in file_set: - pe,pv,pr = dataCache.pkg_pepvpr[f] + pe, pv, pr = dataCache.pkg_pepvpr[f] if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): preferred_file = f preferred_ver = (pe, pv, pr) @@ -136,7 +136,7 @@ def findLatestProvider(pn, cfgData, dataCache, file_set): latest_p = 0 latest_f = None for file_name in file_set: - pe,pv,pr = dataCache.pkg_pepvpr[file_name] + pe, pv, pr = dataCache.pkg_pepvpr[file_name] dp = dataCache.pkg_dp[file_name] if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p): @@ -169,14 +169,14 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): def _filterProviders(providers, item, cfgData, dataCache): """ - Take a list of providers and filter/reorder according to the + Take a list of providers and filter/reorder according to the environment variables and previous build results """ eligible = [] preferred_versions = {} sortpkg_pn = {} - # The order of providers depends on the order of the files on the disk + # The order of providers depends on the order of the files on the disk # up to here. Sort pkg_pn to make dependency issues reproducible rather # than effectively random. providers.sort() @@ -226,7 +226,7 @@ def _filterProviders(providers, item, cfgData, dataCache): def filterProviders(providers, item, cfgData, dataCache): """ - Take a list of providers and filter/reorder according to the + Take a list of providers and filter/reorder according to the environment variables and previous build results Takes a "normal" target item """ @@ -254,7 +254,7 @@ def filterProviders(providers, item, cfgData, dataCache): def filterProvidersRunTime(providers, item, cfgData, dataCache): """ - Take a list of providers and filter/reorder according to the + Take a list of providers and filter/reorder according to the environment variables and previous build results Takes a "runtime" target item """ @@ -297,7 +297,7 @@ def getRuntimeProviders(dataCache, rdepend): rproviders = [] if rdepend in dataCache.rproviders: - rproviders += dataCache.rproviders[rdepend] + rproviders += dataCache.rproviders[rdepend] if rdepend in dataCache.packages: rproviders += dataCache.packages[rdepend] diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index c5f4380c8..a4aea6c00 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -22,15 +22,15 @@ Handles preparation and execution of a queue of tasks # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from bb import msg, data, event, mkdirhier, utils import bb, os, sys +from bb import msg, data, event import signal import stat import fcntl class TaskFailure(Exception): """Exception raised when a task in a runqueue fails""" - def __init__(self, x): + def __init__(self, x): self.args = x @@ -60,7 +60,7 @@ class RunQueueStats: def taskActive(self): self.active = self.active + 1 -# These values indicate the next step due to be run in the +# These values indicate the next step due to be run in the # runQueue state machine runQueuePrepare = 2 runQueueRunInit = 3 @@ -76,7 +76,7 @@ class RunQueueScheduler: """ def __init__(self, runqueue): """ - The default scheduler just returns the first buildable task (the + The default scheduler just returns the first buildable task (the priority map is sorted by task numer) """ self.rq = runqueue @@ -109,8 +109,7 @@ class RunQueueSchedulerSpeed(RunQueueScheduler): self.rq = runqueue - sortweight = deepcopy(self.rq.runq_weight) - sortweight.sort() + sortweight = sorted(deepcopy(self.rq.runq_weight)) copyweight = deepcopy(self.rq.runq_weight) self.prio_map = [] @@ -123,10 +122,10 @@ class RunQueueSchedulerSpeed(RunQueueScheduler): class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): """ - A scheduler optimised to complete .bb files are quickly as possible. The - priority map is sorted by task weight, but then reordered so once a given + A scheduler optimised to complete .bb files are quickly as possible. The + priority map is sorted by task weight, but then reordered so once a given .bb file starts to build, its completed as quickly as possible. This works - well where disk space is at a premium and classes like OE's rm_work are in + well where disk space is at a premium and classes like OE's rm_work are in force. """ def __init__(self, runqueue): @@ -135,7 +134,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): #FIXME - whilst this groups all fnids together it does not reorder the #fnid groups optimally. - + basemap = deepcopy(self.prio_map) self.prio_map = [] while (len(basemap) > 0): @@ -231,7 +230,7 @@ class RunQueue: if chain1[index] != chain2[index]: return False return True - + def chain_array_contains(chain, chain_array): """ Return True if chain_array contains chain @@ -286,7 +285,7 @@ class RunQueue: def calculate_task_weights(self, endpoints): """ - Calculate a number representing the "weight" of each task. Heavier weighted tasks + Calculate a number representing the "weight" of each task. Heavier weighted tasks have more dependencies and hence should be executed sooner for maximum speed. This function also sanity checks the task list finding tasks that its not @@ -307,7 +306,7 @@ class RunQueue: weight[listid] = 1 task_done[listid] = True - while 1: + while True: next_points = [] for listid in endpoints: for revdep in self.runq_depends[listid]: @@ -318,7 +317,7 @@ class RunQueue: task_done[revdep] = True endpoints = next_points if len(next_points) == 0: - break + break # Circular dependency sanity check problem_tasks = [] @@ -345,7 +344,7 @@ class RunQueue: def prepare_runqueue(self): """ - Turn a set of taskData into a RunQueue and compute data needed + Turn a set of taskData into a RunQueue and compute data needed to optimise the execution order. """ @@ -365,12 +364,12 @@ class RunQueue: # Step A - Work out a list of tasks to run # # Taskdata gives us a list of possible providers for every build and run - # target ordered by priority. It also gives information on each of those + # target ordered by priority. It also gives information on each of those # providers. # - # To create the actual list of tasks to execute we fix the list of - # providers and then resolve the dependencies into task IDs. This - # process is repeated for each type of dependency (tdepends, deptask, + # To create the actual list of tasks to execute we fix the list of + # providers and then resolve the dependencies into task IDs. This + # process is repeated for each type of dependency (tdepends, deptask, # rdeptast, recrdeptask, idepends). def add_build_dependencies(depids, tasknames, depends): @@ -411,12 +410,12 @@ class RunQueue: if fnid not in taskData.failed_fnids: - # Resolve task internal dependencies + # Resolve task internal dependencies # # e.g. addtask before X after Y depends = taskData.tasks_tdepends[task] - # Resolve 'deptask' dependencies + # Resolve 'deptask' dependencies # # e.g. do_sometask[deptask] = "do_someothertask" # (makes sure sometask runs after someothertask of all DEPENDS) @@ -424,7 +423,7 @@ class RunQueue: tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() add_build_dependencies(taskData.depids[fnid], tasknames, depends) - # Resolve 'rdeptask' dependencies + # Resolve 'rdeptask' dependencies # # e.g. do_sometask[rdeptask] = "do_someothertask" # (makes sure sometask runs after someothertask of all RDEPENDS) @@ -432,7 +431,7 @@ class RunQueue: taskname = task_deps['rdeptask'][taskData.tasks_name[task]] add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) - # Resolve inter-task dependencies + # Resolve inter-task dependencies # # e.g. do_sometask[depends] = "targetname:do_someothertask" # (makes sure sometask runs after targetname's someothertask) @@ -467,8 +466,8 @@ class RunQueue: newdep = [] bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) for dep in depends: - if task != dep: - newdep.append(dep) + if task != dep: + newdep.append(dep) depends = newdep self.runq_fnid.append(taskData.tasks_fnid[task]) @@ -482,7 +481,7 @@ class RunQueue: # # Build a list of recursive cumulative dependencies for each fnid # We do this by fnid, since if A depends on some task in B - # we're interested in later tasks B's fnid might have but B itself + # we're interested in later tasks B's fnid might have but B itself # doesn't depend on # # Algorithm is O(tasks) + O(tasks)*O(fnids) @@ -513,7 +512,7 @@ class RunQueue: if len(runq_recrdepends[task]) > 0: taskfnid = self.runq_fnid[task] for dep in reccumdepends[taskfnid]: - # Ignore self references + # Ignore self references if dep == task: continue for taskname in runq_recrdepends[task]: @@ -631,11 +630,11 @@ class RunQueue: for dep in revdeps: if dep in self.runq_depends[listid]: #self.dump_data(taskData) - bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep] , taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid])) + bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid])) bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) - # Calculate task weights + # Calculate task weights # Check of higher length circular dependencies self.runq_weight = self.calculate_task_weights(endpoints) @@ -657,7 +656,7 @@ class RunQueue: for prov in self.dataCache.fn_provides[fn]: if prov not in prov_list: prov_list[prov] = [fn] - elif fn not in prov_list[prov]: + elif fn not in prov_list[prov]: prov_list[prov].append(fn) error = False for prov in prov_list: @@ -703,7 +702,7 @@ class RunQueue: buildable.append(task) def check_buildable(self, task, buildable): - for revdep in self.runq_revdeps[task]: + for revdep in self.runq_revdeps[task]: alldeps = 1 for dep in self.runq_depends[revdep]: if dep in unchecked: @@ -774,7 +773,7 @@ class RunQueue: #print "Not current: %s" % notcurrent if len(unchecked) > 0: - bb.fatal("check_stamps fatal internal error") + bb.msg.fatal(bb.msg.domain.RunQueue, "check_stamps fatal internal error") return current def check_stamp_task(self, task): @@ -811,10 +810,10 @@ class RunQueue: try: t2 = os.stat(stampfile2)[stat.ST_MTIME] if t1 < t2: - bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile,stampfile2)) + bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2)) iscurrent = False except: - bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 ,stampfile)) + bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2, stampfile)) iscurrent = False return iscurrent @@ -852,7 +851,7 @@ class RunQueue: return False if self.state is runQueueChildProcess: - print "Child process" + print("Child process") return False # Loop @@ -885,7 +884,7 @@ class RunQueue: def task_complete(self, task): """ Mark a task as completed - Look at the reverse dependencies and mark any task with + Look at the reverse dependencies and mark any task with completed dependencies as buildable """ self.runq_complete[task] = 1 @@ -929,7 +928,7 @@ class RunQueue: while True: task = None if self.stats.active < self.number_tasks: - task = self.sched.next() + task = next(self.sched) if task is not None: fn = self.taskData.fn_index[self.runq_fnid[task]] @@ -948,7 +947,7 @@ class RunQueue: try: pipein, pipeout = os.pipe() pid = os.fork() - except OSError, e: + except OSError as e: bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) if pid == 0: os.close(pipein) @@ -982,10 +981,11 @@ class RunQueue: try: self.cooker.tryBuild(fn, taskname[3:]) except bb.build.EventException: - bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") os._exit(1) - except: - bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") + except Exception: + from traceback import format_exc + bb.msg.error(bb.msg.domain.Build, "Build of %s %s failed" % (fn, taskname)) + bb.msg.error(bb.msg.domain.Build, format_exc()) os._exit(1) os._exit(0) @@ -1031,12 +1031,13 @@ class RunQueue: return def finish_runqueue_now(self): - bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) - for k, v in self.build_pids.iteritems(): - try: - os.kill(-k, signal.SIGINT) - except: - pass + if self.stats.active: + bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) + for k, v in self.build_pids.iteritems(): + try: + os.kill(-k, signal.SIGINT) + except: + pass for pipe in self.build_pipes: self.build_pipes[pipe].read() @@ -1085,30 +1086,30 @@ class RunQueue: """ bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") for task in range(len(self.runq_task)): - bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, - taskQueue.fn_index[self.runq_fnid[task]], - self.runq_task[task], - self.runq_weight[task], - self.runq_depends[task], - self.runq_revdeps[task])) + bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, + taskQueue.fn_index[self.runq_fnid[task]], + self.runq_task[task], + self.runq_weight[task], + self.runq_depends[task], + self.runq_revdeps[task])) bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") for task1 in range(len(self.runq_task)): if task1 in self.prio_map: task = self.prio_map[task1] - bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, - taskQueue.fn_index[self.runq_fnid[task]], - self.runq_task[task], - self.runq_weight[task], - self.runq_depends[task], - self.runq_revdeps[task])) + bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, + taskQueue.fn_index[self.runq_fnid[task]], + self.runq_task[task], + self.runq_weight[task], + self.runq_depends[task], + self.runq_revdeps[task])) class TaskFailure(Exception): """ Exception raised when a task in a runqueue fails """ - def __init__(self, x): + def __init__(self, x): self.args = x @@ -1194,6 +1195,5 @@ class runQueuePipe(): while self.read(): continue if len(self.queue) > 0: - print "Warning, worker left partial message" + print("Warning, worker left partial message") os.close(self.fd) - diff --git a/bitbake/lib/bb/server/none.py b/bitbake/lib/bb/server/none.py index ebda11158..e28aa8d7d 100644 --- a/bitbake/lib/bb/server/none.py +++ b/bitbake/lib/bb/server/none.py @@ -115,7 +115,7 @@ class BitBakeServer(): def register_idle_function(self, function, data): """Register a function to be called while the server is idle""" - assert callable(function) + assert hasattr(function, '__call__') self._idlefuns[function] = data def idle_commands(self, delay): @@ -178,4 +178,3 @@ class BitBakeServerConnection(): self.connection.terminateServer() except: pass - diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py index 3364918c7..cb2949fb9 100644 --- a/bitbake/lib/bb/server/xmlrpc.py +++ b/bitbake/lib/bb/server/xmlrpc.py @@ -42,7 +42,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler import inspect, select if sys.hexversion < 0x020600F0: - print "Sorry, python 2.6 or later is required for bitbake's XMLRPC mode" + print("Sorry, python 2.6 or later is required for bitbake's XMLRPC mode") sys.exit(1) class BitBakeServerCommands(): @@ -74,7 +74,7 @@ class BitBakeServerCommands(): Trigger the server to quit """ self.server.quit = True - print "Server (cooker) exitting" + print("Server (cooker) exitting") return def ping(self): @@ -89,8 +89,8 @@ class BitBakeServer(SimpleXMLRPCServer): def __init__(self, cooker, interface = ("localhost", 0)): """ - Constructor - """ + Constructor + """ SimpleXMLRPCServer.__init__(self, interface, requestHandler=SimpleXMLRPCRequestHandler, logRequests=False, allow_none=True) @@ -112,7 +112,7 @@ class BitBakeServer(SimpleXMLRPCServer): def register_idle_function(self, function, data): """Register a function to be called while the server is idle""" - assert callable(function) + assert hasattr(function, '__call__') self._idlefuns[function] = data def serve_forever(self): @@ -146,7 +146,7 @@ class BitBakeServer(SimpleXMLRPCServer): traceback.print_exc() pass if nextsleep is None and len(self._idlefuns) > 0: - nextsleep = 0 + nextsleep = 0 self.timeout = nextsleep # Tell idle functions we're exiting for function, data in self._idlefuns.items(): @@ -175,7 +175,7 @@ class BitBakeServerConnection(): def terminate(self): # Don't wait for server indefinitely import socket - socket.setdefaulttimeout(2) + socket.setdefaulttimeout(2) try: self.events.system_quit() except: @@ -184,4 +184,3 @@ class BitBakeServerConnection(): self.connection.terminateServer() except: pass - diff --git a/bitbake/lib/bb/shell.py b/bitbake/lib/bb/shell.py index 7abea0f12..f9ca9d5bd 100644 --- a/bitbake/lib/bb/shell.py +++ b/bitbake/lib/bb/shell.py @@ -52,12 +52,14 @@ PROBLEMS: # Import and setup global variables ########################################################################## +from __future__ import print_function +from functools import reduce try: set except NameError: from sets import Set as set -import sys, os, readline, socket, httplib, urllib, commands, popen2, copy, shlex, Queue, fnmatch -from bb import data, parse, build, fatal, cache, taskdata, runqueue, providers as Providers +import sys, os, readline, socket, httplib, urllib, commands, popen2, shlex, Queue, fnmatch +from bb import data, parse, build, cache, taskdata, runqueue, providers as Providers __version__ = "0.5.3.1" __credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de> @@ -98,7 +100,7 @@ class BitBakeShellCommands: def _checkParsed( self ): if not parsed: - print "SHELL: This command needs to parse bbfiles..." + print("SHELL: This command needs to parse bbfiles...") self.parse( None ) def _findProvider( self, item ): @@ -119,28 +121,28 @@ class BitBakeShellCommands: """Register a new name for a command""" new, old = params if not old in cmds: - print "ERROR: Command '%s' not known" % old + print("ERROR: Command '%s' not known" % old) else: cmds[new] = cmds[old] - print "OK" + print("OK") alias.usage = "<alias> <command>" def buffer( self, params ): """Dump specified output buffer""" index = params[0] - print self._shell.myout.buffer( int( index ) ) + print(self._shell.myout.buffer( int( index ) )) buffer.usage = "<index>" def buffers( self, params ): """Show the available output buffers""" commands = self._shell.myout.bufferedCommands() if not commands: - print "SHELL: No buffered commands available yet. Start doing something." + print("SHELL: No buffered commands available yet. Start doing something.") else: - print "="*35, "Available Output Buffers", "="*27 + print("="*35, "Available Output Buffers", "="*27) for index, cmd in enumerate( commands ): - print "| %s %s" % ( str( index ).ljust( 3 ), cmd ) - print "="*88 + print("| %s %s" % ( str( index ).ljust( 3 ), cmd )) + print("="*88) def build( self, params, cmd = "build" ): """Build a providee""" @@ -149,7 +151,7 @@ class BitBakeShellCommands: self._checkParsed() names = globfilter( cooker.status.pkg_pn, globexpr ) if len( names ) == 0: names = [ globexpr ] - print "SHELL: Building %s" % ' '.join( names ) + print("SHELL: Building %s" % ' '.join( names )) td = taskdata.TaskData(cooker.configuration.abort) localdata = data.createCopy(cooker.configuration.data) @@ -168,22 +170,22 @@ class BitBakeShellCommands: tasks.append([name, "do_%s" % cmd]) td.add_unresolved(localdata, cooker.status) - + rq = runqueue.RunQueue(cooker, localdata, cooker.status, td, tasks) rq.prepare_runqueue() rq.execute_runqueue() except Providers.NoProvider: - print "ERROR: No Provider" + print("ERROR: No Provider") last_exception = Providers.NoProvider - except runqueue.TaskFailure, fnids: + except runqueue.TaskFailure as fnids: for fnid in fnids: - print "ERROR: '%s' failed" % td.fn_index[fnid] + print("ERROR: '%s' failed" % td.fn_index[fnid]) last_exception = runqueue.TaskFailure - except build.EventException, e: - print "ERROR: Couldn't build '%s'" % names + except build.EventException as e: + print("ERROR: Couldn't build '%s'" % names) last_exception = e @@ -216,7 +218,7 @@ class BitBakeShellCommands: if bbfile is not None: os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) ) else: - print "ERROR: Nothing provides '%s'" % name + print("ERROR: Nothing provides '%s'" % name) edit.usage = "<providee>" def environment( self, params ): @@ -239,14 +241,14 @@ class BitBakeShellCommands: global last_exception name = params[0] bf = completeFilePath( name ) - print "SHELL: Calling '%s' on '%s'" % ( cmd, bf ) + print("SHELL: Calling '%s' on '%s'" % ( cmd, bf )) try: cooker.buildFile(bf, cmd) except parse.ParseError: - print "ERROR: Unable to open or parse '%s'" % bf - except build.EventException, e: - print "ERROR: Couldn't build '%s'" % name + print("ERROR: Unable to open or parse '%s'" % bf) + except build.EventException as e: + print("ERROR: Couldn't build '%s'" % name) last_exception = e fileBuild.usage = "<bbfile>" @@ -270,62 +272,62 @@ class BitBakeShellCommands: def fileReparse( self, params ): """(re)Parse a bb file""" bbfile = params[0] - print "SHELL: Parsing '%s'" % bbfile + print("SHELL: Parsing '%s'" % bbfile) parse.update_mtime( bbfile ) cooker.bb_cache.cacheValidUpdate(bbfile) fromCache = cooker.bb_cache.loadData(bbfile, cooker.configuration.data, cooker.status) cooker.bb_cache.sync() if False: #fromCache: - print "SHELL: File has not been updated, not reparsing" + print("SHELL: File has not been updated, not reparsing") else: - print "SHELL: Parsed" + print("SHELL: Parsed") fileReparse.usage = "<bbfile>" def abort( self, params ): """Toggle abort task execution flag (see bitbake -k)""" cooker.configuration.abort = not cooker.configuration.abort - print "SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort ) + print("SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort )) def force( self, params ): """Toggle force task execution flag (see bitbake -f)""" cooker.configuration.force = not cooker.configuration.force - print "SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force ) + print("SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force )) def help( self, params ): """Show a comprehensive list of commands and their purpose""" - print "="*30, "Available Commands", "="*30 + print("="*30, "Available Commands", "="*30) for cmd in sorted(cmds): - function,numparams,usage,helptext = cmds[cmd] - print "| %s | %s" % (usage.ljust(30), helptext) - print "="*78 + function, numparams, usage, helptext = cmds[cmd] + print("| %s | %s" % (usage.ljust(30), helptext)) + print("="*78) def lastError( self, params ): """Show the reason or log that was produced by the last BitBake event exception""" if last_exception is None: - print "SHELL: No Errors yet (Phew)..." + print("SHELL: No Errors yet (Phew)...") else: reason, event = last_exception.args - print "SHELL: Reason for the last error: '%s'" % reason + print("SHELL: Reason for the last error: '%s'" % reason) if ':' in reason: msg, filename = reason.split( ':' ) filename = filename.strip() - print "SHELL: Dumping log file for last error:" + print("SHELL: Dumping log file for last error:") try: - print open( filename ).read() + print(open( filename ).read()) except IOError: - print "ERROR: Couldn't open '%s'" % filename + print("ERROR: Couldn't open '%s'" % filename) def match( self, params ): """Dump all files or providers matching a glob expression""" what, globexpr = params if what == "files": self._checkParsed() - for key in globfilter( cooker.status.pkg_fn, globexpr ): print key + for key in globfilter( cooker.status.pkg_fn, globexpr ): print(key) elif what == "providers": self._checkParsed() - for key in globfilter( cooker.status.pkg_pn, globexpr ): print key + for key in globfilter( cooker.status.pkg_pn, globexpr ): print(key) else: - print "Usage: match %s" % self.print_.usage + print("Usage: match %s" % self.print_.usage) match.usage = "<files|providers> <glob>" def new( self, params ): @@ -335,15 +337,15 @@ class BitBakeShellCommands: fulldirname = "%s/%s" % ( packages, dirname ) if not os.path.exists( fulldirname ): - print "SHELL: Creating '%s'" % fulldirname + print("SHELL: Creating '%s'" % fulldirname) os.mkdir( fulldirname ) if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ): if os.path.exists( "%s/%s" % ( fulldirname, filename ) ): - print "SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename ) + print("SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename )) return False - print "SHELL: Creating '%s/%s'" % ( fulldirname, filename ) + print("SHELL: Creating '%s/%s'" % ( fulldirname, filename )) newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" ) - print >>newpackage,"""DESCRIPTION = "" + print("""DESCRIPTION = "" SECTION = "" AUTHOR = "" HOMEPAGE = "" @@ -370,7 +372,7 @@ SRC_URI = "" #do_install() { # #} -""" +""", file=newpackage) newpackage.close() os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) ) new.usage = "<directory> <filename>" @@ -390,14 +392,14 @@ SRC_URI = "" def pasteLog( self, params ): """Send the last event exception error log (if there is one) to http://rafb.net/paste""" if last_exception is None: - print "SHELL: No Errors yet (Phew)..." + print("SHELL: No Errors yet (Phew)...") else: reason, event = last_exception.args - print "SHELL: Reason for the last error: '%s'" % reason + print("SHELL: Reason for the last error: '%s'" % reason) if ':' in reason: msg, filename = reason.split( ':' ) filename = filename.strip() - print "SHELL: Pasting log file to pastebin..." + print("SHELL: Pasting log file to pastebin...") file = open( filename ).read() sendToPastebin( "contents of " + filename, file ) @@ -419,23 +421,23 @@ SRC_URI = "" cooker.buildDepgraph() global parsed parsed = True - print + print() def reparse( self, params ): """(re)Parse a providee's bb file""" bbfile = self._findProvider( params[0] ) if bbfile is not None: - print "SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] ) + print("SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] )) self.fileReparse( [ bbfile ] ) else: - print "ERROR: Nothing provides '%s'" % params[0] + print("ERROR: Nothing provides '%s'" % params[0]) reparse.usage = "<providee>" def getvar( self, params ): """Dump the contents of an outer BitBake environment variable""" var = params[0] value = data.getVar( var, cooker.configuration.data, 1 ) - print value + print(value) getvar.usage = "<variable>" def peek( self, params ): @@ -445,9 +447,9 @@ SRC_URI = "" if bbfile is not None: the_data = cooker.bb_cache.loadDataFull(bbfile, cooker.configuration.data) value = the_data.getVar( var, 1 ) - print value + print(value) else: - print "ERROR: Nothing provides '%s'" % name + print("ERROR: Nothing provides '%s'" % name) peek.usage = "<providee> <variable>" def poke( self, params ): @@ -455,7 +457,7 @@ SRC_URI = "" name, var, value = params bbfile = self._findProvider( name ) if bbfile is not None: - print "ERROR: Sorry, this functionality is currently broken" + print("ERROR: Sorry, this functionality is currently broken") #d = cooker.pkgdata[bbfile] #data.setVar( var, value, d ) @@ -463,7 +465,7 @@ SRC_URI = "" #cooker.pkgdata.setDirty(bbfile, d) #print "OK" else: - print "ERROR: Nothing provides '%s'" % name + print("ERROR: Nothing provides '%s'" % name) poke.usage = "<providee> <variable> <value>" def print_( self, params ): @@ -471,12 +473,12 @@ SRC_URI = "" what = params[0] if what == "files": self._checkParsed() - for key in cooker.status.pkg_fn: print key + for key in cooker.status.pkg_fn: print(key) elif what == "providers": self._checkParsed() - for key in cooker.status.providers: print key + for key in cooker.status.providers: print(key) else: - print "Usage: print %s" % self.print_.usage + print("Usage: print %s" % self.print_.usage) print_.usage = "<files|providers>" def python( self, params ): @@ -496,7 +498,7 @@ SRC_URI = "" """Set an outer BitBake environment variable""" var, value = params data.setVar( var, value, cooker.configuration.data ) - print "OK" + print("OK") setVar.usage = "<variable> <value>" def rebuild( self, params ): @@ -508,7 +510,7 @@ SRC_URI = "" def shell( self, params ): """Execute a shell command and dump the output""" if params != "": - print commands.getoutput( " ".join( params ) ) + print(commands.getoutput( " ".join( params ) )) shell.usage = "<...>" def stage( self, params ): @@ -518,17 +520,17 @@ SRC_URI = "" def status( self, params ): """<just for testing>""" - print "-" * 78 - print "building list = '%s'" % cooker.building_list - print "build path = '%s'" % cooker.build_path - print "consider_msgs_cache = '%s'" % cooker.consider_msgs_cache - print "build stats = '%s'" % cooker.stats - if last_exception is not None: print "last_exception = '%s'" % repr( last_exception.args ) - print "memory output contents = '%s'" % self._shell.myout._buffer + print("-" * 78) + print("building list = '%s'" % cooker.building_list) + print("build path = '%s'" % cooker.build_path) + print("consider_msgs_cache = '%s'" % cooker.consider_msgs_cache) + print("build stats = '%s'" % cooker.stats) + if last_exception is not None: print("last_exception = '%s'" % repr( last_exception.args )) + print("memory output contents = '%s'" % self._shell.myout._buffer) def test( self, params ): """<just for testing>""" - print "testCommand called with '%s'" % params + print("testCommand called with '%s'" % params) def unpack( self, params ): """Execute 'unpack' on a providee""" @@ -553,12 +555,12 @@ SRC_URI = "" try: providers = cooker.status.providers[item] except KeyError: - print "SHELL: ERROR: Nothing provides", preferred + print("SHELL: ERROR: Nothing provides", preferred) else: for provider in providers: if provider == pf: provider = " (***) %s" % provider else: provider = " %s" % provider - print provider + print(provider) which.usage = "<providee>" ########################################################################## @@ -583,7 +585,7 @@ def sendToPastebin( desc, content ): mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" ) mydata["text"] = content params = urllib.urlencode( mydata ) - headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"} + headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} host = "rafb.net" conn = httplib.HTTPConnection( "%s:80" % host ) @@ -594,9 +596,9 @@ def sendToPastebin( desc, content ): if response.status == 302: location = response.getheader( "location" ) or "unknown" - print "SHELL: Pasted to http://%s%s" % ( host, location ) + print("SHELL: Pasted to http://%s%s" % ( host, location )) else: - print "ERROR: %s %s" % ( response.status, response.reason ) + print("ERROR: %s %s" % ( response.status, response.reason )) def completer( text, state ): """Return a possible readline completion""" @@ -643,7 +645,7 @@ def columnize( alist, width = 80 ): return reduce(lambda line, word, width=width: '%s%s%s' % (line, ' \n'[(len(line[line.rfind('\n')+1:]) - + len(word.split('\n',1)[0] + + len(word.split('\n', 1)[0] ) >= width)], word), alist @@ -718,7 +720,7 @@ class BitBakeShell: except IOError: pass # It doesn't exist yet. - print __credits__ + print(__credits__) def cleanup( self ): """Write readline history and clean up resources""" @@ -726,7 +728,7 @@ class BitBakeShell: try: readline.write_history_file( self.historyfilename ) except: - print "SHELL: Unable to save command history" + print("SHELL: Unable to save command history") def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ): """Register a command""" @@ -740,11 +742,11 @@ class BitBakeShell: try: function, numparams, usage, helptext = cmds[command] except KeyError: - print "SHELL: ERROR: '%s' command is not a valid command." % command + print("SHELL: ERROR: '%s' command is not a valid command." % command) self.myout.removeLast() else: if (numparams != -1) and (not len( params ) == numparams): - print "Usage: '%s'" % usage + print("Usage: '%s'" % usage) return result = function( self.commands, params ) @@ -759,7 +761,7 @@ class BitBakeShell: if not cmdline: continue if "|" in cmdline: - print "ERROR: '|' in startup file is not allowed. Ignoring line" + print("ERROR: '|' in startup file is not allowed. Ignoring line") continue self.commandQ.put( cmdline.strip() ) @@ -801,10 +803,10 @@ class BitBakeShell: sys.stdout.write( pipe.fromchild.read() ) # except EOFError: - print + print() return except KeyboardInterrupt: - print + print() ########################################################################## # Start function - called from the BitBake command line utility @@ -819,4 +821,4 @@ def start( aCooker ): bbshell.cleanup() if __name__ == "__main__": - print "SHELL: Sorry, this program should only be called by BitBake." + print("SHELL: Sorry, this program should only be called by BitBake.") diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py index 3e5e006f5..e31f96785 100644 --- a/bitbake/lib/bb/taskdata.py +++ b/bitbake/lib/bb/taskdata.py @@ -34,7 +34,7 @@ def re_match_strings(target, strings): for name in strings: if (name==target or - re.search(name,target)!=None): + re.search(name, target)!=None): return True return False @@ -84,7 +84,7 @@ class TaskData: def getrun_id(self, name): """ - Return an ID number for the run target name. + Return an ID number for the run target name. If it doesn't exist, create one. """ if not name in self.run_names_index: @@ -95,7 +95,7 @@ class TaskData: def getfn_id(self, name): """ - Return an ID number for the filename. + Return an ID number for the filename. If it doesn't exist, create one. """ if not name in self.fn_index: @@ -271,7 +271,7 @@ class TaskData: def get_unresolved_build_targets(self, dataCache): """ - Return a list of build targets who's providers + Return a list of build targets who's providers are unknown. """ unresolved = [] @@ -286,7 +286,7 @@ class TaskData: def get_unresolved_run_targets(self, dataCache): """ - Return a list of runtime targets who's providers + Return a list of runtime targets who's providers are unknown. """ unresolved = [] @@ -304,7 +304,7 @@ class TaskData: Return a list of providers of item """ targetid = self.getbuild_id(item) - + return self.build_targets[targetid] def get_dependees(self, itemid): @@ -354,20 +354,15 @@ class TaskData: self.add_provider_internal(cfgData, dataCache, item) except bb.providers.NoProvider: if self.abort: - if self.get_rdependees_str(item): - bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (item, self.get_dependees_str(item))) - else: - bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (item)) raise - targetid = self.getbuild_id(item) - self.remove_buildtarget(targetid) + self.remove_buildtarget(self.getbuild_id(item)) self.mark_external_target(item) def add_provider_internal(self, cfgData, dataCache, item): """ Add the providers of item to the task data - Mark entries were specifically added externally as against dependencies + Mark entries were specifically added externally as against dependencies added internally during dependency resolution """ @@ -375,11 +370,7 @@ class TaskData: return if not item in dataCache.providers: - if self.get_rdependees_str(item): - bb.msg.note(2, bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (item, self.get_dependees_str(item))) - else: - bb.msg.note(2, bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (item)) - bb.event.fire(bb.event.NoProvider(item), cfgData) + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_rdependees_str(item)), cfgData) raise bb.providers.NoProvider(item) if self.have_build_target(item): @@ -391,8 +382,7 @@ class TaskData: eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] if not eligible: - bb.msg.note(2, bb.msg.domain.Provider, "No buildable provider PROVIDES '%s' but '%s' DEPENDS on or otherwise requires it. Enable debugging and see earlier logs to find unbuildable providers." % (item, self.get_dependees_str(item))) - bb.event.fire(bb.event.NoProvider(item), cfgData) + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item)), cfgData) raise bb.providers.NoProvider(item) if len(eligible) > 1 and foundUnique == False: @@ -400,8 +390,6 @@ class TaskData: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) - bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list))) - bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item) bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData) self.consider_msgs_cache.append(item) @@ -431,16 +419,14 @@ class TaskData: all_p = bb.providers.getRuntimeProviders(dataCache, item) if not all_p: - bb.msg.error(bb.msg.domain.Provider, "'%s' RDEPENDS/RRECOMMENDS or otherwise requires the runtime entity '%s' but it wasn't found in any PACKAGE or RPROVIDES variables" % (self.get_rdependees_str(item), item)) - bb.event.fire(bb.event.NoProvider(item, runtime=True), cfgData) + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item)), cfgData) raise bb.providers.NoRProvider(item) eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] if not eligible: - bb.msg.error(bb.msg.domain.Provider, "'%s' RDEPENDS/RRECOMMENDS or otherwise requires the runtime entity '%s' but it wasn't found in any PACKAGE or RPROVIDES variables of any buildable targets.\nEnable debugging and see earlier logs to find unbuildable targets." % (self.get_rdependees_str(item), item)) - bb.event.fire(bb.event.NoProvider(item, runtime=True), cfgData) + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item)), cfgData) raise bb.providers.NoRProvider(item) if len(eligible) > 1 and numberPreferred == 0: @@ -448,9 +434,7 @@ class TaskData: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) - bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list))) - bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item) - bb.event.fire(bb.event.MultipleProviders(item,providers_list, runtime=True), cfgData) + bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) self.consider_msgs_cache.append(item) if numberPreferred > 1: @@ -458,9 +442,7 @@ class TaskData: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) - bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (top %s entries preferred) (%s);" % (item, numberPreferred, ", ".join(providers_list))) - bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item) - bb.event.fire(bb.event.MultipleProviders(item,providers_list, runtime=True), cfgData) + bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) self.consider_msgs_cache.append(item) # run through the list until we find one that we can build @@ -515,8 +497,9 @@ class TaskData: self.fail_fnid(self.tasks_fnid[taskid], missing_list) if self.abort and targetid in self.external_targets: - bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (self.build_names_index[targetid], missing_list)) - raise bb.providers.NoProvider + target = self.build_names_index[targetid] + bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (target, missing_list)) + raise bb.providers.NoProvider(target) def remove_runtarget(self, targetid, missing_list = []): """ @@ -539,7 +522,7 @@ class TaskData: Resolve all unresolved build and runtime targets """ bb.msg.note(1, bb.msg.domain.TaskData, "Resolving any missing task queue dependencies") - while 1: + while True: added = 0 for target in self.get_unresolved_build_targets(dataCache): try: @@ -548,10 +531,6 @@ class TaskData: except bb.providers.NoProvider: targetid = self.getbuild_id(target) if self.abort and targetid in self.external_targets: - if self.get_rdependees_str(target): - bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s' (but '%s' DEPENDS on or otherwise requires it)" % (target, self.get_dependees_str(target))) - else: - bb.msg.error(bb.msg.domain.Provider, "Nothing PROVIDES '%s'" % (target)) raise self.remove_buildtarget(targetid) for target in self.get_unresolved_run_targets(dataCache): @@ -594,9 +573,9 @@ class TaskData: bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:") for task in range(len(self.tasks_name)): bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % ( - task, - self.fn_index[self.tasks_fnid[task]], - self.tasks_name[task], + task, + self.fn_index[self.tasks_fnid[task]], + self.tasks_name[task], self.tasks_tdepends[task])) bb.msg.debug(3, bb.msg.domain.TaskData, "dependency ids (per fn):") @@ -606,5 +585,3 @@ class TaskData: bb.msg.debug(3, bb.msg.domain.TaskData, "runtime dependency ids (per fn):") for fnid in self.rdepids: bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid])) - - diff --git a/bitbake/lib/bb/ui/__init__.py b/bitbake/lib/bb/ui/__init__.py index c6a377a8e..a4805ed02 100644 --- a/bitbake/lib/bb/ui/__init__.py +++ b/bitbake/lib/bb/ui/__init__.py @@ -15,4 +15,3 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - diff --git a/bitbake/lib/bb/ui/crumbs/__init__.py b/bitbake/lib/bb/ui/crumbs/__init__.py index c6a377a8e..a4805ed02 100644 --- a/bitbake/lib/bb/ui/crumbs/__init__.py +++ b/bitbake/lib/bb/ui/crumbs/__init__.py @@ -15,4 +15,3 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - diff --git a/bitbake/lib/bb/ui/crumbs/buildmanager.py b/bitbake/lib/bb/ui/crumbs/buildmanager.py index f89e8eefd..e858d75e4 100644 --- a/bitbake/lib/bb/ui/crumbs/buildmanager.py +++ b/bitbake/lib/bb/ui/crumbs/buildmanager.py @@ -28,7 +28,7 @@ import time class BuildConfiguration: """ Represents a potential *or* historic *or* concrete build. It encompasses all the things that we need to tell bitbake to do to make it - build what we want it to build. + build what we want it to build. It also stored the metadata URL and the set of possible machines (and the distros / images / uris for these. Apart from the metdata URL these are @@ -73,34 +73,33 @@ class BuildConfiguration: return self.urls # It might be a lot lot better if we stored these in like, bitbake conf - # file format. - @staticmethod + # file format. + @staticmethod def load_from_file (filename): - f = open (filename, "r") conf = BuildConfiguration() - for line in f.readlines(): - data = line.split (";")[1] - if (line.startswith ("metadata-url;")): - conf.metadata_url = data.strip() - continue - if (line.startswith ("url;")): - conf.urls += [data.strip()] - continue - if (line.startswith ("extra-url;")): - conf.extra_urls += [data.strip()] - continue - if (line.startswith ("machine;")): - conf.machine = data.strip() - continue - if (line.startswith ("distribution;")): - conf.distro = data.strip() - continue - if (line.startswith ("image;")): - conf.image = data.strip() - continue + with open(filename, "r") as f: + for line in f: + data = line.split (";")[1] + if (line.startswith ("metadata-url;")): + conf.metadata_url = data.strip() + continue + if (line.startswith ("url;")): + conf.urls += [data.strip()] + continue + if (line.startswith ("extra-url;")): + conf.extra_urls += [data.strip()] + continue + if (line.startswith ("machine;")): + conf.machine = data.strip() + continue + if (line.startswith ("distribution;")): + conf.distro = data.strip() + continue + if (line.startswith ("image;")): + conf.image = data.strip() + continue - f.close () return conf # Serialise to a file. This is part of the build process and we use this @@ -140,13 +139,13 @@ class BuildResult(gobject.GObject): ".conf" in the directory for the build. This is GObject so that it can be included in the TreeStore.""" - + (STATE_COMPLETE, STATE_FAILED, STATE_ONGOING) = \ (0, 1, 2) def __init__ (self, parent, identifier): gobject.GObject.__init__ (self) - self.date = None + self.date = None self.files = [] self.status = None @@ -157,8 +156,8 @@ class BuildResult(gobject.GObject): # format build-<year><month><day>-<ordinal> we can easily # pull it out. # TODO: Better to stat a file? - (_ , date, revision) = identifier.split ("-") - print date + (_, date, revision) = identifier.split ("-") + print(date) year = int (date[0:4]) month = int (date[4:6]) @@ -181,7 +180,7 @@ class BuildResult(gobject.GObject): self.add_file (file) def add_file (self, file): - # Just add the file for now. Don't care about the type. + # Just add the file for now. Don't care about the type. self.files += [(file, None)] class BuildManagerModel (gtk.TreeStore): @@ -194,7 +193,7 @@ class BuildManagerModel (gtk.TreeStore): def __init__ (self): gtk.TreeStore.__init__ (self, - gobject.TYPE_STRING, + gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, @@ -207,7 +206,7 @@ class BuildManager (gobject.GObject): "results" directory but is also used for starting a new build.""" __gsignals__ = { - 'population-finished' : (gobject.SIGNAL_RUN_LAST, + 'population-finished' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()), 'populate-error' : (gobject.SIGNAL_RUN_LAST, @@ -220,13 +219,13 @@ class BuildManager (gobject.GObject): date = long (time.mktime (result.date.timetuple())) # Add a top level entry for the build - - self.model.set (iter, + + self.model.set (iter, BuildManagerModel.COL_IDENT, result.identifier, BuildManagerModel.COL_DESC, result.conf.image, - BuildManagerModel.COL_MACHINE, result.conf.machine, - BuildManagerModel.COL_DISTRO, result.conf.distro, - BuildManagerModel.COL_BUILD_RESULT, result, + BuildManagerModel.COL_MACHINE, result.conf.machine, + BuildManagerModel.COL_DISTRO, result.conf.distro, + BuildManagerModel.COL_BUILD_RESULT, result, BuildManagerModel.COL_DATE, date, BuildManagerModel.COL_STATE, result.state) @@ -257,7 +256,7 @@ class BuildManager (gobject.GObject): while (iter): (ident, state) = self.model.get(iter, - BuildManagerModel.COL_IDENT, + BuildManagerModel.COL_IDENT, BuildManagerModel.COL_STATE) if state == BuildResult.STATE_ONGOING: @@ -385,8 +384,8 @@ class BuildManager (gobject.GObject): build_directory]) server.runCommand(["buildTargets", [conf.image], "rootfs"]) - except Exception, e: - print e + except Exception as e: + print(e) class BuildManagerTreeView (gtk.TreeView): """ The tree view for the build manager. This shows the historic builds @@ -422,29 +421,29 @@ class BuildManagerTreeView (gtk.TreeView): # Misc descriptiony thing renderer = gtk.CellRendererText () - col = gtk.TreeViewColumn (None, renderer, + col = gtk.TreeViewColumn (None, renderer, text=BuildManagerModel.COL_DESC) self.append_column (col) # Machine renderer = gtk.CellRendererText () - col = gtk.TreeViewColumn ("Machine", renderer, + col = gtk.TreeViewColumn ("Machine", renderer, text=BuildManagerModel.COL_MACHINE) self.append_column (col) # distro renderer = gtk.CellRendererText () - col = gtk.TreeViewColumn ("Distribution", renderer, + col = gtk.TreeViewColumn ("Distribution", renderer, text=BuildManagerModel.COL_DISTRO) self.append_column (col) # date (using a custom function for formatting the cell contents it # takes epoch -> human readable string) renderer = gtk.CellRendererText () - col = gtk.TreeViewColumn ("Date", renderer, + col = gtk.TreeViewColumn ("Date", renderer, text=BuildManagerModel.COL_DATE) self.append_column (col) - col.set_cell_data_func (renderer, + col.set_cell_data_func (renderer, self.date_format_custom_cell_data_func) # For status. @@ -454,4 +453,3 @@ class BuildManagerTreeView (gtk.TreeView): self.append_column (col) col.set_cell_data_func (renderer, self.state_format_custom_cell_data_fun) - diff --git a/bitbake/lib/bb/ui/crumbs/runningbuild.py b/bitbake/lib/bb/ui/crumbs/runningbuild.py index 18afd6674..b4416ecbb 100644 --- a/bitbake/lib/bb/ui/crumbs/runningbuild.py +++ b/bitbake/lib/bb/ui/crumbs/runningbuild.py @@ -24,7 +24,7 @@ import gobject class RunningBuildModel (gtk.TreeStore): (COL_TYPE, COL_PACKAGE, COL_TASK, COL_MESSAGE, COL_ICON, COL_ACTIVE) = (0, 1, 2, 3, 4, 5) def __init__ (self): - gtk.TreeStore.__init__ (self, + gtk.TreeStore.__init__ (self, gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, @@ -34,7 +34,7 @@ class RunningBuildModel (gtk.TreeStore): class RunningBuild (gobject.GObject): __gsignals__ = { - 'build-succeeded' : (gobject.SIGNAL_RUN_LAST, + 'build-succeeded' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()), 'build-failed' : (gobject.SIGNAL_RUN_LAST, @@ -63,7 +63,7 @@ class RunningBuild (gobject.GObject): # for the message. if hasattr(event, 'pid'): pid = event.pid - if self.pids_to_task.has_key(pid): + if pid in self.pids_to_task: (package, task) = self.pids_to_task[pid] parent = self.tasks_to_iter[(package, task)] @@ -82,29 +82,29 @@ class RunningBuild (gobject.GObject): # Add the message to the tree either at the top level if parent is # None otherwise as a descendent of a task. - self.model.append (parent, + self.model.append (parent, (event.__name__.split()[-1], # e.g. MsgWarn, MsgError - package, + package, task, event._message, - icon, + icon, False)) elif isinstance(event, bb.build.TaskStarted): (package, task) = (event._package, event._task) # Save out this PID. - self.pids_to_task[pid] = (package,task) + self.pids_to_task[pid] = (package, task) # Check if we already have this package in our model. If so then # that can be the parent for the task. Otherwise we create a new # top level for the package. - if (self.tasks_to_iter.has_key ((package, None))): + if ((package, None) in self.tasks_to_iter): parent = self.tasks_to_iter[(package, None)] else: - parent = self.model.append (None, (None, - package, + parent = self.model.append (None, (None, + package, None, - "Package: %s" % (package), + "Package: %s" % (package), None, False)) self.tasks_to_iter[(package, None)] = parent @@ -114,10 +114,10 @@ class RunningBuild (gobject.GObject): self.model.set(parent, self.model.COL_ICON, "gtk-execute") # Add an entry in the model for this task - i = self.model.append (parent, (None, - package, + i = self.model.append (parent, (None, + package, task, - "Task: %s" % (task), + "Task: %s" % (task), None, False)) @@ -176,5 +176,3 @@ class RunningBuildTreeView (gtk.TreeView): renderer = gtk.CellRendererText () col = gtk.TreeViewColumn ("Message", renderer, text=3) self.append_column (col) - - diff --git a/bitbake/lib/bb/ui/depexp.py b/bitbake/lib/bb/ui/depexp.py index cfa5b6564..1cd58cac1 100644 --- a/bitbake/lib/bb/ui/depexp.py +++ b/bitbake/lib/bb/ui/depexp.py @@ -201,14 +201,14 @@ def init(server, eventHandler): try: cmdline = server.runCommand(["getCmdLineAction"]) if not cmdline or cmdline[0] != "generateDotGraph": - print "This UI is only compatible with the -g option" + print("This UI is only compatible with the -g option") return ret = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]]) if ret != True: - print "Couldn't run command! %s" % ret + print("Couldn't run command! %s" % ret) return - except xmlrpclib.Fault, x: - print "XMLRPC Fault getting commandline:\n %s" % x + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) return shutdown = 0 @@ -233,8 +233,8 @@ def init(server, eventHandler): x = event.sofar y = event.total if x == y: - print("\nParsing finished. %d cached, %d parsed, %d skipped, %d masked, %d errors." - % ( event.cached, event.parsed, event.skipped, event.masked, event.errors)) + print(("\nParsing finished. %d cached, %d parsed, %d skipped, %d masked, %d errors." + % ( event.cached, event.parsed, event.skipped, event.masked, event.errors))) pbar.hide() gtk.gdk.threads_enter() pbar.progress.set_fraction(float(x)/float(y)) @@ -250,7 +250,7 @@ def init(server, eventHandler): if isinstance(event, bb.command.CookerCommandCompleted): continue if isinstance(event, bb.command.CookerCommandFailed): - print "Command execution failed: %s" % event.error + print("Command execution failed: %s" % event.error) break if isinstance(event, bb.cooker.CookerExit): break @@ -259,14 +259,13 @@ def init(server, eventHandler): except KeyboardInterrupt: if shutdown == 2: - print "\nThird Keyboard Interrupt, exit.\n" + print("\nThird Keyboard Interrupt, exit.\n") break if shutdown == 1: - print "\nSecond Keyboard Interrupt, stopping...\n" + print("\nSecond Keyboard Interrupt, stopping...\n") server.runCommand(["stateStop"]) if shutdown == 0: - print "\nKeyboard Interrupt, closing down...\n" + print("\nKeyboard Interrupt, closing down...\n") server.runCommand(["stateShutdown"]) shutdown = shutdown + 1 pass - diff --git a/bitbake/lib/bb/ui/goggle.py b/bitbake/lib/bb/ui/goggle.py index 94995d82d..2cfa002f8 100644 --- a/bitbake/lib/bb/ui/goggle.py +++ b/bitbake/lib/bb/ui/goggle.py @@ -25,13 +25,13 @@ from bb.ui.crumbs.runningbuild import RunningBuildTreeView, RunningBuild def event_handle_idle_func (eventHandler, build): - # Consume as many messages as we can in the time available to us - event = eventHandler.getEvent() - while event: - build.handle_event (event) - event = eventHandler.getEvent() + # Consume as many messages as we can in the time available to us + event = eventHandler.getEvent() + while event: + build.handle_event (event) + event = eventHandler.getEvent() - return True + return True class MainWindow (gtk.Window): def __init__ (self): @@ -55,15 +55,15 @@ def init (server, eventHandler): window.cur_build_tv.set_model (running_build.model) try: cmdline = server.runCommand(["getCmdLineAction"]) - print cmdline + print(cmdline) if not cmdline: return 1 ret = server.runCommand(cmdline) if ret != True: - print "Couldn't get default commandline! %s" % ret + print("Couldn't get default commandline! %s" % ret) return 1 - except xmlrpclib.Fault, x: - print "XMLRPC Fault getting commandline:\n %s" % x + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) return 1 # Use a timeout function for probing the event queue to find out if we @@ -74,4 +74,3 @@ def init (server, eventHandler): running_build) gtk.main() - diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py index ed26bb2b4..f81759abf 100644 --- a/bitbake/lib/bb/ui/knotty.py +++ b/bitbake/lib/bb/ui/knotty.py @@ -18,8 +18,9 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os +from __future__ import division +import os import sys import itertools import xmlrpclib @@ -44,10 +45,10 @@ def init(server, eventHandler): return 1 ret = server.runCommand(cmdline) if ret != True: - print "Couldn't get default commandline! %s" % ret + print("Couldn't get default commandline! %s" % ret) return 1 - except xmlrpclib.Fault, x: - print "XMLRPC Fault getting commandline:\n %s" % x + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) return 1 shutdown = 0 @@ -65,39 +66,39 @@ def init(server, eventHandler): if shutdown and helper.needUpdate: activetasks, failedtasks = helper.getTasks() if activetasks: - print "Waiting for %s active tasks to finish:" % len(activetasks) + print("Waiting for %s active tasks to finish:" % len(activetasks)) tasknum = 1 for task in activetasks: - print "%s: %s (pid %s)" % (tasknum, activetasks[task]["title"], task) + print("%s: %s (pid %s)" % (tasknum, activetasks[task]["title"], task)) tasknum = tasknum + 1 if isinstance(event, bb.msg.MsgPlain): - print event._message + print(event._message) continue if isinstance(event, bb.msg.MsgDebug): - print 'DEBUG: ' + event._message + print('DEBUG: ' + event._message) continue if isinstance(event, bb.msg.MsgNote): - print 'NOTE: ' + event._message + print('NOTE: ' + event._message) continue if isinstance(event, bb.msg.MsgWarn): - print 'WARNING: ' + event._message + print('WARNING: ' + event._message) continue if isinstance(event, bb.msg.MsgError): return_value = 1 - print 'ERROR: ' + event._message + print('ERROR: ' + event._message) continue if isinstance(event, bb.msg.MsgFatal): return_value = 1 - print 'FATAL: ' + event._message + print('FATAL: ' + event._message) break if isinstance(event, bb.build.TaskFailed): return_value = 1 logfile = event.logfile - if logfile: - print "ERROR: Logfile of failure stored in: %s" % logfile + if logfile and os.path.exists(logfile): + print("ERROR: Logfile of failure stored in: %s" % logfile) if 1 or includelogs: - print "Log data follows:" + print("Log data follows:") f = open(logfile, "r") lines = [] while True: @@ -110,19 +111,19 @@ def init(server, eventHandler): if len(lines) > int(loglines): lines.pop(0) else: - print '| %s' % l + print('| %s' % l) f.close() if lines: for line in lines: - print line + print(line) if isinstance(event, bb.build.TaskBase): - print "NOTE: %s" % event._message + print("NOTE: %s" % event._message) continue if isinstance(event, bb.event.ParseProgress): x = event.sofar y = event.total if os.isatty(sys.stdout.fileno()): - sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) ) + sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( next(parsespin), x, y, x*100//y ) ) sys.stdout.flush() else: if x == 1: @@ -132,8 +133,8 @@ def init(server, eventHandler): sys.stdout.write("done.") sys.stdout.flush() if x == y: - print("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." - % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)) + print(("\nParsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors." + % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors))) continue if isinstance(event, bb.command.CookerCommandCompleted): @@ -143,39 +144,48 @@ def init(server, eventHandler): continue if isinstance(event, bb.command.CookerCommandFailed): return_value = 1 - print "Command execution failed: %s" % event.error + print("Command execution failed: %s" % event.error) break if isinstance(event, bb.cooker.CookerExit): break - - # ignore - if isinstance(event, bb.event.BuildStarted): - continue - if isinstance(event, bb.event.BuildCompleted): - continue if isinstance(event, bb.event.MultipleProviders): + print("NOTE: multiple providers are available for %s%s (%s)" % (event._is_runtime and "runtime " or "", + event._item, + ", ".join(event._candidates))) + print("NOTE: consider defining a PREFERRED_PROVIDER entry to match %s" % event._item) + continue + if isinstance(event, bb.event.NoProvider): + if event._runtime: + r = "R" + else: + r = "" + + if event._dependees: + print("ERROR: Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r)) + else: + print("ERROR: Nothing %sPROVIDES '%s'" % (r, event._item)) continue - if isinstance(event, bb.runqueue.runQueueEvent): - continue - if isinstance(event, bb.runqueue.runQueueExitWait): - continue - if isinstance(event, bb.event.StampUpdate): - continue - if isinstance(event, bb.event.ConfigParsed): - continue - if isinstance(event, bb.event.RecipeParsed): + + # ignore + if isinstance(event, (bb.event.BuildBase, + bb.event.StampUpdate, + bb.event.ConfigParsed, + bb.event.RecipeParsed, + bb.runqueue.runQueueEvent, + bb.runqueue.runQueueExitWait)): continue - print "Unknown Event: %s" % event + + print("Unknown Event: %s" % event) except KeyboardInterrupt: if shutdown == 2: - print "\nThird Keyboard Interrupt, exit.\n" + print("\nThird Keyboard Interrupt, exit.\n") break if shutdown == 1: - print "\nSecond Keyboard Interrupt, stopping...\n" + print("\nSecond Keyboard Interrupt, stopping...\n") server.runCommand(["stateStop"]) if shutdown == 0: - print "\nKeyboard Interrupt, closing down...\n" + print("\nKeyboard Interrupt, closing down...\n") server.runCommand(["stateShutdown"]) shutdown = shutdown + 1 pass diff --git a/bitbake/lib/bb/ui/ncurses.py b/bitbake/lib/bb/ui/ncurses.py index 14310dc12..3fed4c58a 100644 --- a/bitbake/lib/bb/ui/ncurses.py +++ b/bitbake/lib/bb/ui/ncurses.py @@ -44,6 +44,8 @@ """ +from __future__ import division + import os, sys, curses, itertools, time import bb import xmlrpclib @@ -136,7 +138,7 @@ class NCursesUI: """Thread Activity Window""" def __init__( self, x, y, width, height ): NCursesUI.DecoratedWindow.__init__( self, "Thread Activity", x, y, width, height ) - + def setStatus( self, thread, text ): line = "%02d: %s" % ( thread, text ) width = self.dimensions[WIDTH] @@ -199,8 +201,8 @@ class NCursesUI: main_left = 0 main_top = 0 - main_height = ( height / 3 * 2 ) - main_width = ( width / 3 ) * 2 + main_height = ( height // 3 * 2 ) + main_width = ( width // 3 ) * 2 clo_left = main_left clo_top = main_top + main_height clo_height = height - main_height - main_top - 1 @@ -225,17 +227,17 @@ class NCursesUI: helper = uihelper.BBUIHelper() shutdown = 0 - + try: cmdline = server.runCommand(["getCmdLineAction"]) if not cmdline: return ret = server.runCommand(cmdline) if ret != True: - print "Couldn't get default commandlind! %s" % ret + print("Couldn't get default commandlind! %s" % ret) return - except xmlrpclib.Fault, x: - print "XMLRPC Fault getting commandline:\n %s" % x + except xmlrpclib.Fault as x: + print("XMLRPC Fault getting commandline:\n %s" % x) return exitflag = False @@ -246,7 +248,7 @@ class NCursesUI: continue helper.eventHandler(event) #mw.appendText("%s\n" % event[0]) - if isinstance(event, bb.build.Task): + if isinstance(event, bb.build.TaskBase): mw.appendText("NOTE: %s\n" % event._message) if isinstance(event, bb.msg.MsgDebug): mw.appendText('DEBUG: ' + event._message + '\n') @@ -263,10 +265,10 @@ class NCursesUI: y = event.total if x == y: mw.setStatus("Idle") - mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked." + mw.appendText("Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( event.cached, event.parsed, event.skipped, event.masked )) else: - mw.setStatus("Parsing: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) ) + mw.setStatus("Parsing: %s (%04d/%04d) [%2d %%]" % ( next(parsespin), x, y, x*100//y ) ) # if isinstance(event, bb.build.TaskFailed): # if event.logfile: # if data.getVar("BBINCLUDELOGS", d): @@ -301,12 +303,12 @@ class NCursesUI: taw.setText(0, 0, "") if activetasks: taw.appendText("Active Tasks:\n") - for task in activetasks: - taw.appendText(task) + for task in activetasks.itervalues(): + taw.appendText(task["title"]) if failedtasks: taw.appendText("Failed Tasks:\n") for task in failedtasks: - taw.appendText(task) + taw.appendText(task["title"]) curses.doupdate() except KeyboardInterrupt: @@ -324,7 +326,7 @@ class NCursesUI: def init(server, eventHandler): if not os.isatty(sys.stdout.fileno()): - print "FATAL: Unable to run 'ncurses' UI without a TTY." + print("FATAL: Unable to run 'ncurses' UI without a TTY.") return ui = NCursesUI() try: @@ -332,4 +334,3 @@ def init(server, eventHandler): except: import traceback traceback.print_exc() - diff --git a/bitbake/lib/bb/ui/puccho.py b/bitbake/lib/bb/ui/puccho.py index 713aa1f4a..a627fc803 100644 --- a/bitbake/lib/bb/ui/puccho.py +++ b/bitbake/lib/bb/ui/puccho.py @@ -24,6 +24,7 @@ import gtk.glade import threading import urllib2 import os +import contextlib from bb.ui.crumbs.buildmanager import BuildManager, BuildConfiguration from bb.ui.crumbs.buildmanager import BuildManagerTreeView @@ -38,7 +39,7 @@ class MetaDataLoader(gobject.GObject): on what machines are available. The distribution and images available for the machine and the the uris to use for building the given machine.""" __gsignals__ = { - 'success' : (gobject.SIGNAL_RUN_LAST, + 'success' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()), 'error' : (gobject.SIGNAL_RUN_LAST, @@ -77,20 +78,19 @@ class MetaDataLoader(gobject.GObject): def run (self): result = {} try: - f = urllib2.urlopen (self.url) - - # Parse the metadata format. The format is.... - # <machine>;<default distro>|<distro>...;<default image>|<image>...;<type##url>|... - for line in f.readlines(): - components = line.split(";") - if (len (components) < 4): - raise MetaDataLoader.LoaderThread.LoaderImportException - machine = components[0] - distros = components[1].split("|") - images = components[2].split("|") - urls = components[3].split("|") - - result[machine] = (distros, images, urls) + with contextlib.closing (urllib2.urlopen (self.url)) as f: + # Parse the metadata format. The format is.... + # <machine>;<default distro>|<distro>...;<default image>|<image>...;<type##url>|... + for line in f: + components = line.split(";") + if (len (components) < 4): + raise MetaDataLoader.LoaderThread.LoaderImportException + machine = components[0] + distros = components[1].split("|") + images = components[2].split("|") + urls = components[3].split("|") + + result[machine] = (distros, images, urls) # Create an object representing this *potential* # configuration. It can become concrete if the machine, distro @@ -104,13 +104,13 @@ class MetaDataLoader(gobject.GObject): gobject.idle_add (MetaDataLoader.emit_success_signal, self.loader) - except MetaDataLoader.LoaderThread.LoaderImportException, e: + except MetaDataLoader.LoaderThread.LoaderImportException as e: gobject.idle_add (MetaDataLoader.emit_error_signal, self.loader, "Repository metadata corrupt") - except Exception, e: + except Exception as e: gobject.idle_add (MetaDataLoader.emit_error_signal, self.loader, "Unable to download repository metadata") - print e + print(e) def try_fetch_from_url (self, url): # Try and download the metadata. Firing a signal if successful @@ -211,7 +211,7 @@ class BuildSetupDialog (gtk.Dialog): # Build button = gtk.Button ("_Build", None, True) image = gtk.Image () - image.set_from_stock (gtk.STOCK_EXECUTE,gtk.ICON_SIZE_BUTTON) + image.set_from_stock (gtk.STOCK_EXECUTE, gtk.ICON_SIZE_BUTTON) button.set_image (image) self.add_action_widget (button, BuildSetupDialog.RESPONSE_BUILD) button.show_all () @@ -293,7 +293,7 @@ class BuildSetupDialog (gtk.Dialog): if (active_iter): self.configuration.machine = model.get(active_iter, 0)[0] - # Extract the chosen distro from the combo + # Extract the chosen distro from the combo model = self.distribution_combo.get_model() active_iter = self.distribution_combo.get_active_iter() if (active_iter): @@ -311,62 +311,62 @@ class BuildSetupDialog (gtk.Dialog): # # TODO: Should be a method on the RunningBuild class def event_handle_timeout (eventHandler, build): - # Consume as many messages as we can ... - event = eventHandler.getEvent() - while event: - build.handle_event (event) - event = eventHandler.getEvent() - return True + # Consume as many messages as we can ... + event = eventHandler.getEvent() + while event: + build.handle_event (event) + event = eventHandler.getEvent() + return True class MainWindow (gtk.Window): - # Callback that gets fired when the user hits a button in the - # BuildSetupDialog. - def build_dialog_box_response_cb (self, dialog, response_id): - conf = None - if (response_id == BuildSetupDialog.RESPONSE_BUILD): - dialog.update_configuration() - print dialog.configuration.machine, dialog.configuration.distro, \ - dialog.configuration.image - conf = dialog.configuration + # Callback that gets fired when the user hits a button in the + # BuildSetupDialog. + def build_dialog_box_response_cb (self, dialog, response_id): + conf = None + if (response_id == BuildSetupDialog.RESPONSE_BUILD): + dialog.update_configuration() + print(dialog.configuration.machine, dialog.configuration.distro, \ + dialog.configuration.image) + conf = dialog.configuration - dialog.destroy() + dialog.destroy() - if conf: - self.manager.do_build (conf) + if conf: + self.manager.do_build (conf) - def build_button_clicked_cb (self, button): - dialog = BuildSetupDialog () + def build_button_clicked_cb (self, button): + dialog = BuildSetupDialog () - # For some unknown reason Dialog.run causes nice little deadlocks ... :-( - dialog.connect ("response", self.build_dialog_box_response_cb) - dialog.show() + # For some unknown reason Dialog.run causes nice little deadlocks ... :-( + dialog.connect ("response", self.build_dialog_box_response_cb) + dialog.show() - def __init__ (self): - gtk.Window.__init__ (self) + def __init__ (self): + gtk.Window.__init__ (self) - # Pull in *just* the main vbox from the Glade XML data and then pack - # that inside the window - gxml = gtk.glade.XML (os.path.dirname(__file__) + "/crumbs/puccho.glade", - root = "main_window_vbox") - vbox = gxml.get_widget ("main_window_vbox") - self.add (vbox) + # Pull in *just* the main vbox from the Glade XML data and then pack + # that inside the window + gxml = gtk.glade.XML (os.path.dirname(__file__) + "/crumbs/puccho.glade", + root = "main_window_vbox") + vbox = gxml.get_widget ("main_window_vbox") + self.add (vbox) - # Create the tree views for the build manager view and the progress view - self.build_manager_view = BuildManagerTreeView() - self.running_build_view = RunningBuildTreeView() + # Create the tree views for the build manager view and the progress view + self.build_manager_view = BuildManagerTreeView() + self.running_build_view = RunningBuildTreeView() - # Grab the scrolled windows that we put the tree views into - self.results_scrolledwindow = gxml.get_widget ("results_scrolledwindow") - self.progress_scrolledwindow = gxml.get_widget ("progress_scrolledwindow") + # Grab the scrolled windows that we put the tree views into + self.results_scrolledwindow = gxml.get_widget ("results_scrolledwindow") + self.progress_scrolledwindow = gxml.get_widget ("progress_scrolledwindow") - # Put the tree views inside ... - self.results_scrolledwindow.add (self.build_manager_view) - self.progress_scrolledwindow.add (self.running_build_view) + # Put the tree views inside ... + self.results_scrolledwindow.add (self.build_manager_view) + self.progress_scrolledwindow.add (self.running_build_view) - # Hook up the build button... - self.build_button = gxml.get_widget ("main_toolbutton_build") - self.build_button.connect ("clicked", self.build_button_clicked_cb) + # Hook up the build button... + self.build_button = gxml.get_widget ("main_toolbutton_build") + self.build_button.connect ("clicked", self.build_button_clicked_cb) # I'm not very happy about the current ownership of the RunningBuild. I have # my suspicions that this object should be held by the BuildManager since we @@ -383,11 +383,11 @@ def running_build_succeeded_cb (running_build, manager): # BuildManager. It can then hook onto the signals directly and drive # interesting things it cares about. manager.notify_build_succeeded () - print "build succeeded" + print("build succeeded") def running_build_failed_cb (running_build, manager): # As above - print "build failed" + print("build failed") manager.notify_build_failed () def init (server, eventHandler): diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py index 36302f4da..f1e4d791e 100644 --- a/bitbake/lib/bb/ui/uievent.py +++ b/bitbake/lib/bb/ui/uievent.py @@ -19,7 +19,7 @@ """ -Use this class to fork off a thread to recieve event callbacks from the bitbake +Use this class to fork off a thread to recieve event callbacks from the bitbake server and queue them for the UI to process. This process must be used to avoid client/server deadlocks. """ @@ -110,16 +110,15 @@ class UIXMLRPCServer (SimpleXMLRPCServer): return (sock, addr) except socket.timeout: pass - return (None,None) + return (None, None) def close_request(self, request): if request is None: return SimpleXMLRPCServer.close_request(self, request) - + def process_request(self, request, client_address): if request is None: return SimpleXMLRPCServer.process_request(self, request, client_address) - diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 86b9c724e..c0cc9c6ea 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -19,10 +19,22 @@ BitBake Utility Functions # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import re, fcntl, os, string, stat, shutil, time +import sys +import bb +import errno +import bb.msg +from commands import getstatusoutput + +# Version comparison separators = ".-" -import re, fcntl, os, types, bb, string, stat, shutil -from commands import getstatusoutput +# Context used in better_exec, eval +_context = { + "os": os, + "bb": bb, + "time": time, +} def explode_version(s): r = [] @@ -60,9 +72,9 @@ def vercmp_part(a, b): if ca == None and cb == None: return 0 - if type(ca) is types.StringType: + if isinstance(ca, basestring): sa = ca in separators - if type(cb) is types.StringType: + if isinstance(cb, basestring): sb = cb in separators if sa and not sb: return -1 @@ -85,6 +97,131 @@ def vercmp(ta, tb): r = vercmp_part(ra, rb) return r +_package_weights_ = {"pre":-2, "p":0, "alpha":-4, "beta":-3, "rc":-1} # dicts are unordered +_package_ends_ = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ] # so we need ordered list + +def relparse(myver): + """Parses the last elements of a version number into a triplet, that can + later be compared. + """ + + number = 0 + p1 = 0 + p2 = 0 + mynewver = myver.split('_') + if len(mynewver) == 2: + # an _package_weights_ + number = float(mynewver[0]) + match = 0 + for x in _package_ends_: + elen = len(x) + if mynewver[1][:elen] == x: + match = 1 + p1 = _package_weights_[x] + try: + p2 = float(mynewver[1][elen:]) + except: + p2 = 0 + break + if not match: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + # letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + else: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + #letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + return [number, p1, p2] + +__vercmp_cache__ = {} + +def vercmp_string(val1, val2): + """This takes two version strings and returns an integer to tell you whether + the versions are the same, val1>val2 or val2>val1. + """ + + # quick short-circuit + if val1 == val2: + return 0 + valkey = val1 + " " + val2 + + # cache lookup + try: + return __vercmp_cache__[valkey] + try: + return - __vercmp_cache__[val2 + " " + val1] + except KeyError: + pass + except KeyError: + pass + + # consider 1_p2 vc 1.1 + # after expansion will become (1_p2,0) vc (1,1) + # then 1_p2 is compared with 1 before 0 is compared with 1 + # to solve the bug we need to convert it to (1,0_p2) + # by splitting _prepart part and adding it back _after_expansion + + val1_prepart = val2_prepart = '' + if val1.count('_'): + val1, val1_prepart = val1.split('_', 1) + if val2.count('_'): + val2, val2_prepart = val2.split('_', 1) + + # replace '-' by '.' + # FIXME: Is it needed? can val1/2 contain '-'? + + val1 = val1.split("-") + if len(val1) == 2: + val1[0] = val1[0] + "." + val1[1] + val2 = val2.split("-") + if len(val2) == 2: + val2[0] = val2[0] + "." + val2[1] + + val1 = val1[0].split('.') + val2 = val2[0].split('.') + + # add back decimal point so that .03 does not become "3" ! + for x in range(1, len(val1)): + if val1[x][0] == '0' : + val1[x] = '.' + val1[x] + for x in range(1, len(val2)): + if val2[x][0] == '0' : + val2[x] = '.' + val2[x] + + # extend varion numbers + if len(val2) < len(val1): + val2.extend(["0"]*(len(val1)-len(val2))) + elif len(val1) < len(val2): + val1.extend(["0"]*(len(val2)-len(val1))) + + # add back _prepart tails + if val1_prepart: + val1[-1] += '_' + val1_prepart + if val2_prepart: + val2[-1] += '_' + val2_prepart + # The above code will extend version numbers out so they + # have the same number of digits. + for x in range(0, len(val1)): + cmp1 = relparse(val1[x]) + cmp2 = relparse(val2[x]) + for y in range(0, 3): + myret = cmp1[y] - cmp2[y] + if myret != 0: + __vercmp_cache__[valkey] = myret + return myret + __vercmp_cache__[valkey] = 0 + return 0 + def explode_deps(s): """ Take an RDEPENDS style string of format: @@ -154,26 +291,22 @@ def _print_trace(body, line): """ Print the Environment of a Text Body """ - import bb - # print the environment of the method bb.msg.error(bb.msg.domain.Util, "Printing the environment of the function") - min_line = max(1,line-4) - max_line = min(line+4,len(body)-1) - for i in range(min_line,max_line+1): + min_line = max(1, line-4) + max_line = min(line + 4, len(body)-1) + for i in range(min_line, max_line + 1): bb.msg.error(bb.msg.domain.Util, "\t%.4d:%s" % (i, body[i-1]) ) -def better_compile(text, file, realfile): +def better_compile(text, file, realfile, mode = "exec"): """ A better compile method. This method will print the offending lines. """ try: - return compile(text, file, "exec") - except Exception, e: - import bb,sys - + return compile(text, file, mode) + except Exception as e: # split the text into lines again body = text.split('\n') bb.msg.error(bb.msg.domain.Util, "Error in compiling python function in: ", realfile) @@ -191,18 +324,18 @@ def better_exec(code, context, text, realfile): print the lines that are responsible for the error. """ - import bb,sys + import bb.parse try: - exec code in context + exec(code, _context, context) except: - (t,value,tb) = sys.exc_info() + (t, value, tb) = sys.exc_info() if t in [bb.parse.SkipPackage, bb.build.FuncFailed]: raise # print the Header of the Error Message bb.msg.error(bb.msg.domain.Util, "Error in executing python function in: %s" % realfile) - bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t,value) ) + bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t, value)) # let us find the line number now while tb.tb_next: @@ -212,48 +345,14 @@ def better_exec(code, context, text, realfile): line = traceback.tb_lineno(tb) _print_trace( text.split('\n'), line ) - + raise -def Enum(*names): - """ - A simple class to give Enum support - """ - - assert names, "Empty enums are not supported" - - class EnumClass(object): - __slots__ = names - def __iter__(self): return iter(constants) - def __len__(self): return len(constants) - def __getitem__(self, i): return constants[i] - def __repr__(self): return 'Enum' + str(names) - def __str__(self): return 'enum ' + str(constants) - - class EnumValue(object): - __slots__ = ('__value') - def __init__(self, value): self.__value = value - Value = property(lambda self: self.__value) - EnumType = property(lambda self: EnumType) - def __hash__(self): return hash(self.__value) - def __cmp__(self, other): - # C fans might want to remove the following assertion - # to make all enums comparable by ordinal value {;)) - assert self.EnumType is other.EnumType, "Only values from the same enum are comparable" - return cmp(self.__value, other.__value) - def __invert__(self): return constants[maximum - self.__value] - def __nonzero__(self): return bool(self.__value) - def __repr__(self): return str(names[self.__value]) - - maximum = len(names) - 1 - constants = [None] * len(names) - for i, each in enumerate(names): - val = EnumValue(i) - setattr(EnumClass, each, val) - constants[i] = val - constants = tuple(constants) - EnumType = EnumClass() - return EnumType +def simple_exec(code, context): + exec(code, _context, context) + +def better_eval(source, locals): + return eval(source, _context, locals) def lockfile(name): """ @@ -262,37 +361,36 @@ def lockfile(name): """ path = os.path.dirname(name) if not os.path.isdir(path): - import bb, sys bb.msg.error(bb.msg.domain.Util, "Error, lockfile path does not exist!: %s" % path) sys.exit(1) while True: # If we leave the lockfiles lying around there is no problem # but we should clean up after ourselves. This gives potential - # for races though. To work around this, when we acquire the lock - # we check the file we locked was still the lock file on disk. - # by comparing inode numbers. If they don't match or the lockfile + # for races though. To work around this, when we acquire the lock + # we check the file we locked was still the lock file on disk. + # by comparing inode numbers. If they don't match or the lockfile # no longer exists, we start again. - # This implementation is unfair since the last person to request the + # This implementation is unfair since the last person to request the # lock is the most likely to win it. try: - lf = open(name, "a+") + lf = open(name, "a + ") fcntl.flock(lf.fileno(), fcntl.LOCK_EX) statinfo = os.fstat(lf.fileno()) if os.path.exists(lf.name): - statinfo2 = os.stat(lf.name) - if statinfo.st_ino == statinfo2.st_ino: - return lf + statinfo2 = os.stat(lf.name) + if statinfo.st_ino == statinfo2.st_ino: + return lf # File no longer exists or changed, retry lf.close - except Exception, e: + except Exception as e: continue def unlockfile(lf): """ - Unlock a file locked using lockfile() + Unlock a file locked using lockfile() """ os.unlink(lf.name) fcntl.flock(lf.fileno(), fcntl.LOCK_UN) @@ -308,7 +406,7 @@ def md5_file(filename): except ImportError: import md5 m = md5.new() - + for line in open(filename): m.update(line) return m.hexdigest() @@ -368,19 +466,17 @@ def filter_environment(good_vars): are not known and may influence the build in a negative way. """ - import bb - removed_vars = [] for key in os.environ.keys(): if key in good_vars: continue - + removed_vars.append(key) os.unsetenv(key) del os.environ[key] if len(removed_vars): - bb.debug(1, "Removed the following variables from the environment:", ",".join(removed_vars)) + bb.msg.debug(1, bb.msg.domain.Util, "Removed the following variables from the environment:", ",".join(removed_vars)) return removed_vars @@ -410,7 +506,7 @@ def build_environment(d): """ Build an environment from all exported variables. """ - import bb + import bb.data for var in bb.data.keys(d): export = bb.data.getVarFlag(var, "export", d) if export: @@ -419,7 +515,7 @@ def build_environment(d): def prunedir(topdir): # Delete everything reachable from the directory named in 'topdir'. # CAUTION: This is dangerous! - for root, dirs, files in os.walk(topdir, topdown=False): + for root, dirs, files in os.walk(topdir, topdown = False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: @@ -434,7 +530,7 @@ def prunedir(topdir): # but thats possibly insane and suffixes is probably going to be small # def prune_suffix(var, suffixes, d): - # See if var ends with any of the suffixes listed and + # See if var ends with any of the suffixes listed and # remove it if found for suffix in suffixes: if var.endswith(suffix): @@ -446,169 +542,167 @@ def mkdirhier(dir): directory already exists like os.makedirs """ - bb.debug(3, "mkdirhier(%s)" % dir) + bb.msg.debug(3, bb.msg.domain.Util, "mkdirhier(%s)" % dir) try: os.makedirs(dir) - bb.debug(2, "created " + dir) - except OSError, e: - if e.errno != 17: raise e - -import stat + bb.msg.debug(2, bb.msg.domain.Util, "created " + dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise e -def movefile(src,dest,newmtime=None,sstat=None): +def movefile(src, dest, newmtime = None, sstat = None): """Moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. Move is atomic. """ - #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")" + #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")" try: if not sstat: - sstat=os.lstat(src) - except Exception, e: - print "movefile: Stating source file failed...", e + sstat = os.lstat(src) + except Exception as e: + print("movefile: Stating source file failed...", e) return None - destexists=1 + destexists = 1 try: - dstat=os.lstat(dest) + dstat = os.lstat(dest) except: - dstat=os.lstat(os.path.dirname(dest)) - destexists=0 + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 if destexists: if stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) - destexists=0 - except Exception, e: + destexists = 0 + except Exception as e: pass if stat.S_ISLNK(sstat[stat.ST_MODE]): try: - target=os.readlink(src) + target = os.readlink(src) if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) - os.symlink(target,dest) + os.symlink(target, dest) #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) os.unlink(src) return os.lstat(dest) - except Exception, e: - print "movefile: failed to properly create symlink:", dest, "->", target, e + except Exception as e: + print("movefile: failed to properly create symlink:", dest, "->", target, e) return None - renamefailed=1 - if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]: + renamefailed = 1 + if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]: try: - ret=os.rename(src,dest) - renamefailed=0 - except Exception, e: - import errno - if e[0]!=errno.EXDEV: + os.rename(src, dest) + renamefailed = 0 + except Exception as e: + if e[0] != errno.EXDEV: # Some random error. - print "movefile: Failed to move", src, "to", dest, e + print("movefile: Failed to move", src, "to", dest, e) return None # Invalid cross-device-link 'bind' mounted or actually Cross-Device if renamefailed: - didcopy=0 + didcopy = 0 if stat.S_ISREG(sstat[stat.ST_MODE]): try: # For safety copy then move it over. - shutil.copyfile(src,dest+"#new") - os.rename(dest+"#new",dest) - didcopy=1 - except Exception, e: - print 'movefile: copy', src, '->', dest, 'failed.', e + shutil.copyfile(src, dest + "#new") + os.rename(dest + "#new", dest) + didcopy = 1 + except Exception as e: + print('movefile: copy', src, '->', dest, 'failed.', e) return None else: #we don't yet handle special, so we need to fall back to /bin/mv - a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'") - if a[0]!=0: - print "movefile: Failed to move special file:" + src + "' to '" + dest + "'", a + a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'") + if a[0] != 0: + print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a) return None # failure try: if didcopy: - os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown os.unlink(src) - except Exception, e: - print "movefile: Failed to chown/chmod/unlink", dest, e + except Exception as e: + print("movefile: Failed to chown/chmod/unlink", dest, e) return None if newmtime: - os.utime(dest,(newmtime,newmtime)) + os.utime(dest, (newmtime, newmtime)) else: os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) - newmtime=sstat[stat.ST_MTIME] + newmtime = sstat[stat.ST_MTIME] return newmtime -def copyfile(src,dest,newmtime=None,sstat=None): +def copyfile(src, dest, newmtime = None, sstat = None): """ Copies a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. """ - #print "copyfile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")" + #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")" try: if not sstat: - sstat=os.lstat(src) - except Exception, e: - print "copyfile: Stating source file failed...", e + sstat = os.lstat(src) + except Exception as e: + print("copyfile: Stating source file failed...", e) return False - destexists=1 + destexists = 1 try: - dstat=os.lstat(dest) + dstat = os.lstat(dest) except: - dstat=os.lstat(os.path.dirname(dest)) - destexists=0 + dstat = os.lstat(os.path.dirname(dest)) + destexists = 0 if destexists: if stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) - destexists=0 - except Exception, e: + destexists = 0 + except Exception as e: pass if stat.S_ISLNK(sstat[stat.ST_MODE]): try: - target=os.readlink(src) + target = os.readlink(src) if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) - os.symlink(target,dest) + os.symlink(target, dest) #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) return os.lstat(dest) - except Exception, e: - print "copyfile: failed to properly create symlink:", dest, "->", target, e + except Exception as e: + print("copyfile: failed to properly create symlink:", dest, "->", target, e) return False if stat.S_ISREG(sstat[stat.ST_MODE]): - try: # For safety copy then move it over. - shutil.copyfile(src,dest+"#new") - os.rename(dest+"#new",dest) - except Exception, e: - print 'copyfile: copy', src, '->', dest, 'failed.', e - return False + try: # For safety copy then move it over. + shutil.copyfile(src, dest + "#new") + os.rename(dest + "#new", dest) + except Exception as e: + print('copyfile: copy', src, '->', dest, 'failed.', e) + return False else: - #we don't yet handle special, so we need to fall back to /bin/mv - a=getstatusoutput("/bin/cp -f "+"'"+src+"' '"+dest+"'") - if a[0]!=0: - print "copyfile: Failed to copy special file:" + src + "' to '" + dest + "'", a - return False # failure + #we don't yet handle special, so we need to fall back to /bin/mv + a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'") + if a[0] != 0: + print("copyfile: Failed to copy special file:" + src + "' to '" + dest + "'", a) + return False # failure try: - os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown - except Exception, e: - print "copyfile: Failed to chown/chmod/unlink", dest, e + except Exception as e: + print("copyfile: Failed to chown/chmod/unlink", dest, e) return False if newmtime: - os.utime(dest,(newmtime,newmtime)) + os.utime(dest, (newmtime, newmtime)) else: os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) - newmtime=sstat[stat.ST_MTIME] + newmtime = sstat[stat.ST_MTIME] return newmtime def which(path, item, direction = 0): |