summaryrefslogtreecommitdiff
path: root/bitbake/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/COW.py305
-rw-r--r--bitbake/lib/bb/__init__.py27
-rw-r--r--bitbake/lib/bb/build.py110
-rw-r--r--bitbake/lib/bb/cache.py207
-rw-r--r--bitbake/lib/bb/data.py49
-rw-r--r--bitbake/lib/bb/data_smart.py43
-rw-r--r--bitbake/lib/bb/fetch/__init__.py167
-rw-r--r--bitbake/lib/bb/fetch/cvs.py255
-rw-r--r--bitbake/lib/bb/fetch/git.py144
-rw-r--r--bitbake/lib/bb/fetch/local.py18
-rw-r--r--bitbake/lib/bb/fetch/perforce.py213
-rw-r--r--bitbake/lib/bb/fetch/ssh.py94
-rw-r--r--bitbake/lib/bb/fetch/svk.py160
-rw-r--r--bitbake/lib/bb/fetch/svn.py203
-rw-r--r--bitbake/lib/bb/fetch/wget.py152
-rw-r--r--bitbake/lib/bb/methodpool.py3
-rw-r--r--bitbake/lib/bb/msg.py108
-rw-r--r--bitbake/lib/bb/parse/__init__.py11
-rw-r--r--bitbake/lib/bb/parse/parse_c/BBHandler.py151
-rw-r--r--bitbake/lib/bb/parse/parse_c/Makefile16
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakec.pyx205
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakeparser.cc144
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakescanner.cc218
-rw-r--r--bitbake/lib/bb/parse/parse_c/bitbakescanner.l30
-rw-r--r--bitbake/lib/bb/parse/parse_c/lexer.h8
-rw-r--r--bitbake/lib/bb/parse/parse_c/lexerc.h2
-rw-r--r--bitbake/lib/bb/parse/parse_py/BBHandler.py28
-rw-r--r--bitbake/lib/bb/parse/parse_py/ConfHandler.py33
-rw-r--r--bitbake/lib/bb/providers.py209
-rw-r--r--bitbake/lib/bb/runqueue.py491
-rw-r--r--bitbake/lib/bb/shell.py113
-rw-r--r--bitbake/lib/bb/taskdata.py558
-rw-r--r--bitbake/lib/bb/utils.py54
33 files changed, 3343 insertions, 1186 deletions
diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py
new file mode 100644
index 000000000..826d435f9
--- /dev/null
+++ b/bitbake/lib/bb/COW.py
@@ -0,0 +1,305 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+This is a copy on write dictionary and set which abuses classes to try and be nice and fast.
+
+Please Note:
+ Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
+ Assign a file to __warn__ to get warnings about slow operations.
+"""
+
+from inspect import getmro
+
+import copy
+import types, sets
+types.ImmutableTypes = tuple([ \
+ types.BooleanType, \
+ types.ComplexType, \
+ types.FloatType, \
+ types.IntType, \
+ types.LongType, \
+ types.NoneType, \
+ types.TupleType, \
+ sets.ImmutableSet] + \
+ list(types.StringTypes))
+
+MUTABLE = "__mutable__"
+
+class COWMeta(type):
+ pass
+
+class COWDictMeta(COWMeta):
+ __warn__ = False
+ __hasmutable__ = False
+ __marker__ = tuple()
+
+ def __str__(cls):
+ # FIXME: I have magic numbers!
+ return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+ __repr__ = __str__
+
+ def cow(cls):
+ class C(cls):
+ __count__ = cls.__count__ + 1
+ return C
+ copy = cow
+ __call__ = cow
+
+ def __setitem__(cls, key, value):
+ if not isinstance(value, types.ImmutableTypes):
+ if not isinstance(value, COWMeta):
+ cls.__hasmutable__ = True
+ key += MUTABLE
+ setattr(cls, key, value)
+
+ def __getmutable__(cls, key, readonly=False):
+ nkey = key + MUTABLE
+ try:
+ return cls.__dict__[nkey]
+ except KeyError:
+ pass
+
+ value = getattr(cls, nkey)
+ if readonly:
+ return value
+
+ if not cls.__warn__ is False and not isinstance(value, COWMeta):
+ print >> cls.__warn__, "Warning: Doing a copy because %s is a mutable type." % key
+ try:
+ value = value.copy()
+ except AttributeError, e:
+ value = copy.copy(value)
+ setattr(cls, nkey, value)
+ return value
+
+ __getmarker__ = []
+ def __getreadonly__(cls, key, default=__getmarker__):
+ """\
+ Get a value (even if mutable) which you promise not to change.
+ """
+ return cls.__getitem__(key, default, True)
+
+ def __getitem__(cls, key, default=__getmarker__, readonly=False):
+ try:
+ try:
+ value = getattr(cls, key)
+ except AttributeError:
+ value = cls.__getmutable__(key, readonly)
+
+ # This is for values which have been deleted
+ if value is cls.__marker__:
+ raise AttributeError("key %s does not exist." % key)
+
+ return value
+ except AttributeError, e:
+ if not default is cls.__getmarker__:
+ return default
+
+ raise KeyError(str(e))
+
+ def __delitem__(cls, key):
+ cls.__setitem__(key, cls.__marker__)
+
+ def __revertitem__(cls, key):
+ if not cls.__dict__.has_key(key):
+ key += MUTABLE
+ delattr(cls, key)
+
+ def has_key(cls, key):
+ value = cls.__getreadonly__(key, cls.__marker__)
+ if value is cls.__marker__:
+ return False
+ return True
+
+ def iter(cls, type, readonly=False):
+ for key in dir(cls):
+ if key.startswith("__"):
+ continue
+
+ if key.endswith(MUTABLE):
+ key = key[:-len(MUTABLE)]
+
+ if type == "keys":
+ yield key
+
+ try:
+ if readonly:
+ value = cls.__getreadonly__(key)
+ else:
+ value = cls[key]
+ except KeyError:
+ continue
+
+ if type == "values":
+ yield value
+ if type == "items":
+ yield (key, value)
+ raise StopIteration()
+
+ def iterkeys(cls):
+ return cls.iter("keys")
+ def itervalues(cls, readonly=False):
+ if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
+ print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
+ return cls.iter("values", readonly)
+ def iteritems(cls, readonly=False):
+ if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
+ print >> cls.__warn__, "Warning: If you arn't going to change any of the values call with True."
+ return cls.iter("items", readonly)
+
+class COWSetMeta(COWDictMeta):
+ def __str__(cls):
+ # FIXME: I have magic numbers!
+ return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) -3)
+ __repr__ = __str__
+
+ def cow(cls):
+ class C(cls):
+ __count__ = cls.__count__ + 1
+ return C
+
+ def add(cls, value):
+ COWDictMeta.__setitem__(cls, repr(hash(value)), value)
+
+ def remove(cls, value):
+ COWDictMeta.__delitem__(cls, repr(hash(value)))
+
+ def __in__(cls, value):
+ return COWDictMeta.has_key(repr(hash(value)))
+
+ def iterkeys(cls):
+ raise TypeError("sets don't have keys")
+
+ def iteritems(cls):
+ raise TypeError("sets don't have 'items'")
+
+# These are the actual classes you use!
+class COWDictBase(object):
+ __metaclass__ = COWDictMeta
+ __count__ = 0
+
+class COWSetBase(object):
+ __metaclass__ = COWSetMeta
+ __count__ = 0
+
+if __name__ == "__main__":
+ import sys
+ COWDictBase.__warn__ = sys.stderr
+ a = COWDictBase()
+ print "a", a
+
+ a['a'] = 'a'
+ a['b'] = 'b'
+ a['dict'] = {}
+
+ b = a.copy()
+ print "b", b
+ b['c'] = 'b'
+
+ print
+
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems():
+ print x
+ print
+
+ b['dict']['a'] = 'b'
+ b['a'] = 'c'
+
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems():
+ print x
+ print
+
+ try:
+ b['dict2']
+ except KeyError, e:
+ print "Okay!"
+
+ a['set'] = COWSetBase()
+ a['set'].add("o1")
+ a['set'].add("o1")
+ a['set'].add("o2")
+
+ print "a", a
+ for x in a['set'].itervalues():
+ print x
+ print "--"
+ print "b", b
+ for x in b['set'].itervalues():
+ print x
+ print
+
+ b['set'].add('o3')
+
+ print "a", a
+ for x in a['set'].itervalues():
+ print x
+ print "--"
+ print "b", b
+ for x in b['set'].itervalues():
+ print x
+ print
+
+ a['set2'] = set()
+ a['set2'].add("o1")
+ a['set2'].add("o1")
+ a['set2'].add("o2")
+
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems(readonly=True):
+ print x
+ print
+
+ del b['b']
+ try:
+ print b['b']
+ except KeyError:
+ print "Yay! deleted key raises error"
+
+ if b.has_key('b'):
+ print "Boo!"
+ else:
+ print "Yay - has_key with delete works!"
+
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems(readonly=True):
+ print x
+ print
+
+ b.__revertitem__('b')
+
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems(readonly=True):
+ print x
+ print
+
+ b.__revertitem__('dict')
+ print "a", a
+ for x in a.iteritems():
+ print x
+ print "--"
+ print "b", b
+ for x in b.iteritems(readonly=True):
+ print x
+ print
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index c3e7a1665..61eb5f3db 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -23,7 +23,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA.
"""
-__version__ = "1.4.3"
+__version__ = "1.7.4"
__all__ = [
@@ -63,24 +63,24 @@ __all__ = [
"manifest",
"methodpool",
"cache",
+ "runqueue",
+ "taskdata",
+ "providers",
]
whitespace = '\t\n\x0b\x0c\r '
lowercase = 'abcdefghijklmnopqrstuvwxyz'
-import sys, os, types, re, string
+import sys, os, types, re, string, bb
+from bb import msg
#projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
projectdir = os.getcwd()
-debug_level = 0
-
if "BBDEBUG" in os.environ:
level = int(os.environ["BBDEBUG"])
if level:
- debug_level = level
- else:
- debug_level = 0
+ bb.msg.set_debug_level(level)
class VarExpandError(Exception):
pass
@@ -99,22 +99,17 @@ class MalformedUrl(Exception):
#######################################################################
#######################################################################
-debug_prepend = ''
-
-
def debug(lvl, *args):
- if debug_level >= lvl:
- print debug_prepend + 'DEBUG:', ''.join(args)
+ bb.msg.std_debug(lvl, ''.join(args))
def note(*args):
- print debug_prepend + 'NOTE:', ''.join(args)
+ bb.msg.std_note(''.join(args))
def error(*args):
- print debug_prepend + 'ERROR:', ''.join(args)
+ bb.msg.std_error(''.join(args))
def fatal(*args):
- print debug_prepend + 'ERROR:', ''.join(args)
- sys.exit(1)
+ bb.msg.std_fatal(''.join(args))
#######################################################################
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 8e169e002..942bdc1a3 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -25,18 +25,9 @@ You should have received a copy of the GNU General Public License along with
Based on functions from the base bb module, Copyright 2003 Holger Schurig
"""
-from bb import debug, data, fetch, fatal, error, note, event, mkdirhier, utils
+from bb import data, fetch, event, mkdirhier, utils
import bb, os
-# data holds flags and function name for a given task
-_task_data = data.init()
-
-# graph represents task interdependencies
-_task_graph = bb.digraph()
-
-# stack represents execution order, excepting dependencies
-_task_stack = []
-
# events
class FuncFailed(Exception):
"""Executed function failed"""
@@ -76,13 +67,6 @@ class InvalidTask(TaskBase):
# functions
-def init(data):
- global _task_data, _task_graph, _task_stack
- _task_data = data.init()
- _task_graph = bb.digraph()
- _task_stack = []
-
-
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
@@ -163,7 +147,7 @@ def exec_func_shell(func, d):
f = open(runfile, "w")
f.write("#!/bin/sh -e\n")
- if bb.debug_level > 0: f.write("set -x\n")
+ if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
data.emit_env(f, d)
f.write("cd %s\n" % os.getcwd())
@@ -171,18 +155,18 @@ def exec_func_shell(func, d):
f.close()
os.chmod(runfile, 0775)
if not func:
- error("Function not specified")
+ bb.msg.error(bb.msg.domain.Build, "Function not specified")
raise FuncFailed()
# open logs
si = file('/dev/null', 'r')
try:
- if bb.debug_level > 0:
+ if bb.msg.debug_level['default'] > 0:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
- bb.error("opening log file: %s" % e)
+ bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
se = so
@@ -205,7 +189,10 @@ def exec_func_shell(func, d):
else:
maybe_fakeroot = ''
ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile))
- os.chdir(prevdir)
+ try:
+ os.chdir(prevdir)
+ except:
+ pass
if not interact:
# restore the backups
@@ -224,14 +211,14 @@ def exec_func_shell(func, d):
os.close(ose[0])
if ret==0:
- if bb.debug_level > 0:
+ if bb.msg.debug_level['default'] > 0:
os.remove(runfile)
# os.remove(logfile)
return
else:
- error("function %s failed" % func)
+ bb.msg.error(bb.msg.domain.Build, "function %s failed" % func)
if data.getVar("BBINCLUDELOGS", d):
- error("log data follows (%s)" % logfile)
+ bb.msg.error(bb.msg.domain.Build, "log data follows (%s)" % logfile)
f = open(logfile, "r")
while True:
l = f.readline()
@@ -241,7 +228,7 @@ def exec_func_shell(func, d):
print '| %s' % l
f.close()
else:
- error("see log in %s" % logfile)
+ bb.msg.error(bb.msg.domain.Build, "see log in %s" % logfile)
raise FuncFailed( logfile )
@@ -281,7 +268,7 @@ def exec_task(task, d):
return 1
try:
- debug(1, "Executing task %s" % item)
+ bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % item)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
@@ -292,21 +279,63 @@ def exec_task(task, d):
task_cache.append(item)
data.setVar('_task_cache', task_cache, d)
except FuncFailed, reason:
- note( "Task failed: %s" % reason )
+ bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason )
failedevent = TaskFailed(item, d)
event.fire(failedevent)
raise EventException("Function failed in task: %s" % reason, failedevent)
- # execute
- task_graph.walkdown(task, execute)
+ if data.getVarFlag(task, 'dontrundeps', d):
+ execute(None, task)
+ else:
+ task_graph.walkdown(task, execute)
# make stamp, or cause event and raise exception
if not data.getVarFlag(task, 'nostamp', d):
mkstamp(task, d)
+def stamp_is_current_cache(dataCache, file_name, task, checkdeps = 1):
+ """
+ Check status of a given task's stamp.
+ Returns 0 if it is not current and needs updating.
+ Same as stamp_is_current but works against the dataCache instead of d
+ """
+ task_graph = dataCache.task_queues[file_name]
+
+ if not dataCache.stamp[file_name]:
+ return 0
+
+ stampfile = "%s.%s" % (dataCache.stamp[file_name], task)
+ if not os.access(stampfile, os.F_OK):
+ return 0
+
+ if checkdeps == 0:
+ return 1
+
+ import stat
+ tasktime = os.stat(stampfile)[stat.ST_MTIME]
+
+ _deps = []
+ def checkStamp(graph, task):
+ # check for existance
+ if 'nostamp' in dataCache.task_deps[file_name] and task in dataCache.task_deps[file_name]['nostamp']:
+ return 1
+
+ if not stamp_is_current_cache(dataCache, file_name, task, 0):
+ return 0
+
+ depfile = "%s.%s" % (dataCache.stamp[file_name], task)
+ deptime = os.stat(depfile)[stat.ST_MTIME]
+ if deptime > tasktime:
+ return 0
+ return 1
+
+ return task_graph.walkdown(task, checkStamp)
def stamp_is_current(task, d, checkdeps = 1):
- """Check status of a given task's stamp. returns 0 if it is not current and needs updating."""
+ """
+ Check status of a given task's stamp.
+ Returns 0 if it is not current and needs updating.
+ """
task_graph = data.getVar('_task_graph', d)
if not task_graph:
task_graph = bb.digraph()
@@ -360,7 +389,6 @@ def mkstamp(task, d):
f = open(stamp, "w")
f.close()
-
def add_task(task, deps, d):
task_graph = data.getVar('_task_graph', d)
if not task_graph:
@@ -374,6 +402,21 @@ def add_task(task, deps, d):
# don't assume holding a reference
data.setVar('_task_graph', task_graph, d)
+ task_deps = data.getVar('_task_deps', d)
+ if not task_deps:
+ task_deps = {}
+ def getTask(name):
+ deptask = data.getVarFlag(task, name, d)
+ if deptask:
+ if not name in task_deps:
+ task_deps[name] = {}
+ task_deps[name][task] = deptask
+ getTask('deptask')
+ getTask('rdeptask')
+ getTask('recrdeptask')
+ getTask('nostamp')
+
+ data.setVar('_task_deps', task_deps, d)
def remove_task(task, kill, d):
"""Remove an BB 'task'.
@@ -399,6 +442,3 @@ def task_exists(task, d):
task_graph = bb.digraph()
data.setVar('_task_graph', task_graph, d)
return task_graph.hasnode(task)
-
-def get_task_data():
- return _task_data
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 921a9f758..05c42518a 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -33,15 +33,15 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
import os, re
import bb.data
import bb.utils
+from sets import Set
try:
import cPickle as pickle
except ImportError:
import pickle
- print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
+ bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
-# __cache_version__ = "123"
-__cache_version__ = "124" # changes the __depends structure
+__cache_version__ = "125"
class Cache:
"""
@@ -58,14 +58,12 @@ class Cache:
if self.cachedir in [None, '']:
self.has_cache = False
- if cooker.cb is not None:
- print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
+ bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.")
else:
self.has_cache = True
self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
- if cooker.cb is not None:
- print "NOTE: Using cache in '%s'" % self.cachedir
+ bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
try:
os.stat( self.cachedir )
except OSError:
@@ -80,7 +78,7 @@ class Cache:
if version_data['BITBAKE_VER'] != bb.__version__:
raise ValueError, 'Bitbake Version Mismatch'
except (ValueError, KeyError):
- bb.note("Invalid cache found, rebuilding...")
+ bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
self.depends_cache = {}
if self.depends_cache:
@@ -108,7 +106,7 @@ class Cache:
if fn != self.data_fn:
# We're trying to access data in the cache which doesn't exist
# yet setData hasn't been called to setup the right access. Very bad.
- bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
+ bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
result = bb.data.getVar(var, self.data, exp)
self.depends_cache[fn][var] = result
@@ -127,15 +125,15 @@ class Cache:
self.getVar("__depends", fn, True)
self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
- def loadDataFull(self, fn, cooker):
+ def loadDataFull(self, fn, cfgData):
"""
Return a complete set of data for fn.
To do this, we need to parse the file.
"""
- bb_data, skipped = self.load_bbfile(fn, cooker)
+ bb_data, skipped = self.load_bbfile(fn, cfgData)
return bb_data
- def loadData(self, fn, cooker):
+ def loadData(self, fn, cfgData):
"""
Load a subset of data for fn.
If the cached data is valid we do nothing,
@@ -148,7 +146,7 @@ class Cache:
return True, True
return True, False
- bb_data, skipped = self.load_bbfile(fn, cooker)
+ bb_data, skipped = self.load_bbfile(fn, cfgData)
self.setData(fn, bb_data)
return False, skipped
@@ -175,32 +173,36 @@ class Cache:
# Check file still exists
if self.mtime(fn) == 0:
- bb.debug(2, "Cache: %s not longer exists" % fn)
+ bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
self.remove(fn)
return False
# File isn't in depends_cache
if not fn in self.depends_cache:
- bb.debug(2, "Cache: %s is not cached" % fn)
+ bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn)
self.remove(fn)
return False
# Check the file's timestamp
if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
- bb.debug(2, "Cache: %s changed" % fn)
+ bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
self.remove(fn)
return False
# Check dependencies are still valid
depends = self.getVar("__depends", fn, True)
for f,old_mtime in depends:
+ # Check if file still exists
+ if self.mtime(f) == 0:
+ return False
+
new_mtime = bb.parse.cached_mtime(f)
if (new_mtime > old_mtime):
- bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f))
+ bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
self.remove(fn)
return False
- bb.debug(2, "Depends Cache: %s is clean" % fn)
+ bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
if not fn in self.clean:
self.clean[fn] = ""
@@ -220,7 +222,7 @@ class Cache:
Remove a fn from the cache
Called from the parser in error cases
"""
- bb.debug(1, "Removing %s from cache" % fn)
+ bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn)
if fn in self.depends_cache:
del self.depends_cache[fn]
if fn in self.clean:
@@ -229,7 +231,7 @@ class Cache:
def sync(self):
"""
Save the cache
- Called from the parser when complete (or exitting)
+ Called from the parser when complete (or exiting)
"""
if not self.has_cache:
@@ -243,12 +245,103 @@ class Cache:
p.dump([self.depends_cache, version_data])
def mtime(self, cachefile):
- try:
- return os.stat(cachefile)[8]
- except OSError:
- return 0
+ return bb.parse.cached_mtime_noerror(cachefile)
- def load_bbfile( self, bbfile , cooker):
+ def handle_data(self, file_name, cacheData):
+ """
+ Save data we need into the cache
+ """
+
+ pn = self.getVar('PN', file_name, True)
+ pv = self.getVar('PV', file_name, True)
+ pr = self.getVar('PR', file_name, True)
+ dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
+ provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
+ depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
+ packages = (self.getVar('PACKAGES', file_name, True) or "").split()
+ packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
+ rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
+
+ cacheData.task_queues[file_name] = self.getVar("_task_graph", file_name, True)
+ cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True)
+
+ # build PackageName to FileName lookup table
+ if pn not in cacheData.pkg_pn:
+ cacheData.pkg_pn[pn] = []
+ cacheData.pkg_pn[pn].append(file_name)
+
+ cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
+
+ # build FileName to PackageName lookup table
+ cacheData.pkg_fn[file_name] = pn
+ cacheData.pkg_pvpr[file_name] = (pv,pr)
+ cacheData.pkg_dp[file_name] = dp
+
+ # Build forward and reverse provider hashes
+ # Forward: virtual -> [filenames]
+ # Reverse: PN -> [virtuals]
+ if pn not in cacheData.pn_provides:
+ cacheData.pn_provides[pn] = Set()
+ cacheData.pn_provides[pn] |= provides
+
+ for provide in provides:
+ if provide not in cacheData.providers:
+ cacheData.providers[provide] = []
+ cacheData.providers[provide].append(file_name)
+
+ cacheData.deps[file_name] = Set()
+ for dep in depends:
+ cacheData.all_depends.add(dep)
+ cacheData.deps[file_name].add(dep)
+
+ # Build reverse hash for PACKAGES, so runtime dependencies
+ # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
+ for package in packages:
+ if not package in cacheData.packages:
+ cacheData.packages[package] = []
+ cacheData.packages[package].append(file_name)
+ rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
+
+ for package in packages_dynamic:
+ if not package in cacheData.packages_dynamic:
+ cacheData.packages_dynamic[package] = []
+ cacheData.packages_dynamic[package].append(file_name)
+
+ for rprovide in rprovides:
+ if not rprovide in cacheData.rproviders:
+ cacheData.rproviders[rprovide] = []
+ cacheData.rproviders[rprovide].append(file_name)
+
+ # Build hash of runtime depends and rececommends
+
+ def add_dep(deplist, deps):
+ for dep in deps:
+ if not dep in deplist:
+ deplist[dep] = ""
+
+ if not file_name in cacheData.rundeps:
+ cacheData.rundeps[file_name] = {}
+ if not file_name in cacheData.runrecs:
+ cacheData.runrecs[file_name] = {}
+
+ for package in packages + [pn]:
+ if not package in cacheData.rundeps[file_name]:
+ cacheData.rundeps[file_name][package] = {}
+ if not package in cacheData.runrecs[file_name]:
+ cacheData.runrecs[file_name][package] = {}
+
+ add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or ""))
+ add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or ""))
+ add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
+ add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
+
+ # Collect files we may need for possible world-dep
+ # calculations
+ if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
+ cacheData.possible_world.append(file_name)
+
+
+ def load_bbfile( self, bbfile , config):
"""
Load and parse one .bb build file
Return the data and whether parsing resulted in the file being skipped
@@ -257,25 +350,15 @@ class Cache:
import bb
from bb import utils, data, parse, debug, event, fatal
- topdir = data.getVar('TOPDIR', cooker.configuration.data)
- if not topdir:
- topdir = os.path.abspath(os.getcwd())
- # set topdir to here
- data.setVar('TOPDIR', topdir, cooker.configuration)
- bbfile = os.path.abspath(bbfile)
- bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
# expand tmpdir to include this topdir
- data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data)
- # set topdir to location of .bb file
- topdir = bbfile_loc
- #data.setVar('TOPDIR', topdir, cfg)
- # go there
+ data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
+ bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
oldpath = os.path.abspath(os.getcwd())
- if self.mtime(topdir):
- os.chdir(topdir)
- bb_data = data.init_db(cooker.configuration.data)
+ if self.mtime(bbfile_loc):
+ os.chdir(bbfile_loc)
+ bb_data = data.init_db(config)
try:
- parse.handle(bbfile, bb_data) # read .bb data
+ bb_data = parse.handle(bbfile, bb_data) # read .bb data
os.chdir(oldpath)
return bb_data, False
except bb.parse.SkipPackage:
@@ -304,3 +387,45 @@ def init(cooker):
"""
return Cache(cooker)
+
+
+#============================================================================#
+# CacheData
+#============================================================================#
+class CacheData:
+ """
+ The data structures we compile from the cached data
+ """
+
+ def __init__(self):
+ """
+ Direct cache variables
+ (from Cache.handle_data)
+ """
+ self.providers = {}
+ self.rproviders = {}
+ self.packages = {}
+ self.packages_dynamic = {}
+ self.possible_world = []
+ self.pkg_pn = {}
+ self.pkg_fn = {}
+ self.pkg_pvpr = {}
+ self.pkg_dp = {}
+ self.pn_provides = {}
+ self.all_depends = Set()
+ self.deps = {}
+ self.rundeps = {}
+ self.runrecs = {}
+ self.task_queues = {}
+ self.task_deps = {}
+ self.stamp = {}
+ self.preferred = {}
+
+ """
+ Indirect Cache variables
+ (set elsewhere)
+ """
+ self.ignored_dependencies = []
+ self.world_target = Set()
+ self.bbfile_priority = {}
+ self.bbfile_config_priorities = []
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 55d1cc905..819dff967 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -45,7 +45,8 @@ else:
path = os.path.dirname(os.path.dirname(sys.argv[0]))
sys.path.insert(0,path)
-from bb import note, debug, data_smart
+from bb import data_smart
+import bb
_dict_type = data_smart.DataSmart
@@ -362,10 +363,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
val.rstrip()
if not val:
return 0
+
+ varExpanded = expand(var, d)
if getVarFlag(var, "func", d):
# NOTE: should probably check for unbalanced {} within the var
- o.write("%s() {\n%s\n}\n" % (var, val))
+ o.write("%s() {\n%s\n}\n" % (varExpanded, val))
else:
if getVarFlag(var, "export", d):
o.write('export ')
@@ -375,7 +378,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
# if we're going to output this within doublequotes,
# to a shell, we need to escape the quotes in the var
alter = re.sub('"', '\\"', val.strip())
- o.write('%s="%s"\n' % (var, alter))
+ o.write('%s="%s"\n' % (varExpanded, alter))
return 1
@@ -430,8 +433,38 @@ def update_data(d):
>>> update_data(d)
>>> print getVar('TEST', d)
local
+
+ CopyMonster:
+ >>> e = d.createCopy()
+ >>> setVar('TEST_foo', 'foo', e)
+ >>> update_data(e)
+ >>> print getVar('TEST', e)
+ local
+
+ >>> setVar('OVERRIDES', 'arm:ramses:local:foo', e)
+ >>> update_data(e)
+ >>> print getVar('TEST', e)
+ foo
+
+ >>> f = d.createCopy()
+ >>> setVar('TEST_moo', 'something', f)
+ >>> setVar('OVERRIDES', 'moo:arm:ramses:local:foo', e)
+ >>> update_data(e)
+ >>> print getVar('TEST', e)
+ foo
+
+
+ >>> h = init()
+ >>> setVar('SRC_URI', 'file://append.foo;patch=1 ', h)
+ >>> g = h.createCopy()
+ >>> setVar('SRC_URI_append_arm', 'file://other.foo;patch=1', g)
+ >>> setVar('OVERRIDES', 'arm:moo', g)
+ >>> update_data(g)
+ >>> print getVar('SRC_URI', g)
+ file://append.foo;patch=1 file://other.foo;patch=1
+
"""
- debug(2, "update_data()")
+ bb.msg.debug(2, bb.msg.domain.Data, "update_data()")
# now ask the cookie monster for help
#print "Cookie Monster"
@@ -460,7 +493,7 @@ def update_data(d):
l = len(o)+1
# see if one should even try
- if not o in d._seen_overrides:
+ if not d._seen_overrides.has_key(o):
continue
vars = d._seen_overrides[o]
@@ -469,10 +502,10 @@ def update_data(d):
try:
d[name] = d[var]
except:
- note ("Untracked delVar")
+ bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar")
# now on to the appends and prepends
- if '_append' in d._special_values:
+ if d._special_values.has_key('_append'):
appends = d._special_values['_append'] or []
for append in appends:
for (a, o) in getVarFlag(append, '_append', d) or []:
@@ -487,7 +520,7 @@ def update_data(d):
setVar(append, sval, d)
- if '_prepend' in d._special_values:
+ if d._special_values.has_key('_prepend'):
prepends = d._special_values['_prepend'] or []
for prepend in prepends:
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index fbd4167fe..054b85220 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -29,14 +29,12 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig
"""
import copy, os, re, sys, time, types
-from bb import note, debug, error, fatal, utils, methodpool
+import bb
+from bb import utils, methodpool
+from COW import COWDictBase
from sets import Set
+from new import classobj
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
- print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
__setvar_keyword__ = ["_append","_prepend"]
__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?')
@@ -45,12 +43,14 @@ __expand_python_regexp__ = re.compile(r"\${@.+?}")
class DataSmart:
- def __init__(self):
+ def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ):
self.dict = {}
# cookie monster tribute
- self._special_values = {}
- self._seen_overrides = {}
+ self._special_values = special
+ self._seen_overrides = seen
+
+ self.expand_cache = {}
def expand(self,s, varname):
def var_sub(match):
@@ -75,6 +75,9 @@ class DataSmart:
if type(s) is not types.StringType: # sanity check
return s
+ if varname and varname in self.expand_cache:
+ return self.expand_cache[varname]
+
while s.find('$') != -1:
olds = s
try:
@@ -82,15 +85,20 @@ class DataSmart:
s = __expand_python_regexp__.sub(python_sub, s)
if s == olds: break
if type(s) is not types.StringType: # sanity check
- error('expansion of %s returned non-string %s' % (olds, s))
+ bb.msg.error(bb.msg.domain.Data, 'expansion of %s returned non-string %s' % (olds, s))
except KeyboardInterrupt:
raise
except:
- note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s))
+ bb.msg.note(1, bb.msg.domain.Data, "%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s))
raise
+
+ if varname:
+ self.expand_cache[varname] = s
+
return s
def initVar(self, var):
+ self.expand_cache = {}
if not var in self.dict:
self.dict[var] = {}
@@ -119,6 +127,7 @@ class DataSmart:
self.initVar(var)
def setVar(self,var,value):
+ self.expand_cache = {}
match = __setvar_regexp__.match(var)
if match and match.group("keyword") in __setvar_keyword__:
base = match.group('base')
@@ -128,6 +137,7 @@ class DataSmart:
l.append([value, override])
self.setVarFlag(base, keyword, l)
+ # todo make sure keyword is not __doc__ or __module__
# pay the cookie monster
try:
self._special_values[keyword].add( base )
@@ -135,10 +145,6 @@ class DataSmart:
self._special_values[keyword] = Set()
self._special_values[keyword].add( base )
- # SRC_URI_append_simpad is both a flag and a override
- #if not override in self._seen_overrides:
- # self._seen_overrides[override] = Set()
- #self._seen_overrides[override].add( base )
return
if not var in self.dict:
@@ -150,7 +156,7 @@ class DataSmart:
# more cookies for the cookie monster
if '_' in var:
override = var[var.rfind('_')+1:]
- if not override in self._seen_overrides:
+ if not self._seen_overrides.has_key(override):
self._seen_overrides[override] = Set()
self._seen_overrides[override].add( var )
@@ -165,6 +171,7 @@ class DataSmart:
return value
def delVar(self,var):
+ self.expand_cache = {}
self.dict[var] = {}
def setVarFlag(self,var,flag,flagvalue):
@@ -234,10 +241,8 @@ class DataSmart:
Create a copy of self by setting _data to self
"""
# we really want this to be a DataSmart...
- data = DataSmart()
+ data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
data.dict["_data"] = self.dict
- data._seen_overrides = copy.deepcopy(self._seen_overrides)
- data._special_values = copy.deepcopy(self._special_values)
return data
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index 7ab059076..24aebc41c 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -38,13 +38,16 @@ class NoMethodError(Exception):
class MissingParameterError(Exception):
"""Exception raised when a fetch method is missing a critical parameter in the url"""
+class ParameterError(Exception):
+ """Exception raised when a url cannot be proccessed due to invalid parameters."""
+
class MD5SumError(Exception):
"""Exception raised when a MD5SUM of a file does not match the expected one"""
def uri_replace(uri, uri_find, uri_replace, d):
-# bb.note("uri_replace: operating on %s" % uri)
+# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
if not uri or not uri_find or not uri_replace:
- bb.debug(1, "uri_replace: passed an undefined value, not replacing")
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
uri_decoded = list(bb.decodeurl(uri))
uri_find_decoded = list(bb.decodeurl(uri_find))
uri_replace_decoded = list(bb.decodeurl(uri_replace))
@@ -62,9 +65,9 @@ def uri_replace(uri, uri_find, uri_replace, d):
localfn = bb.fetch.localpath(uri, d)
if localfn:
result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
-# bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
+# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
else:
-# bb.note("uri_replace: no match")
+# bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match")
return uri
# else:
# for j in i.keys():
@@ -72,62 +75,94 @@ def uri_replace(uri, uri_find, uri_replace, d):
return bb.encodeurl(result_decoded)
methods = []
+urldata = {}
def init(urls = [], d = None):
if d == None:
- bb.debug(2,"BUG init called with None as data object!!!")
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "BUG init called with None as data object!!!")
return
for m in methods:
m.urls = []
for u in urls:
+ ud = initdata(u, d)
+ if ud.method:
+ ud.method.urls.append(u)
+
+def initdata(url, d):
+ if url not in urldata:
+ ud = FetchData()
+ (ud.type, ud.host, ud.path, ud.user, ud.pswd, ud.parm) = bb.decodeurl(data.expand(url, d))
+ ud.date = Fetch.getSRCDate(d)
for m in methods:
- m.data = d
- if m.supports(u, d):
- m.urls.append(u)
+ if m.supports(url, ud, d):
+ ud.localpath = m.localpath(url, ud, d)
+ ud.md5 = ud.localpath + '.md5'
+ # if user sets localpath for file, use it instead.
+ if "localpath" in ud.parm:
+ ud.localpath = ud.parm["localpath"]
+ ud.method = m
+ break
+ urldata[url] = ud
+ return urldata[url]
def go(d):
"""Fetch all urls"""
for m in methods:
- if m.urls:
- m.go(d)
+ for u in m.urls:
+ ud = urldata[u]
+ if ud.localfile and not m.forcefetch(u, ud, d) and os.path.exists(urldata[u].md5):
+ # File already present along with md5 stamp file
+ # Touch md5 file to show activity
+ os.utime(ud.md5, None)
+ continue
+ # RP - is olddir needed?
+ # olddir = os.path.abspath(os.getcwd())
+ m.go(u, ud , d)
+ # os.chdir(olddir)
+ if ud.localfile and not m.forcefetch(u, ud, d):
+ Fetch.write_md5sum(u, ud, d)
def localpaths(d):
"""Return a list of the local filenames, assuming successful fetch"""
local = []
for m in methods:
for u in m.urls:
- local.append(m.localpath(u, d))
+ local.append(urldata[u].localpath)
return local
def localpath(url, d):
- for m in methods:
- if m.supports(url, d):
- return m.localpath(url, d)
+ ud = initdata(url, d)
+ if ud.method:
+ return ud.localpath
return url
+class FetchData(object):
+ """Class for fetcher variable store"""
+ def __init__(self):
+ self.localfile = ""
+
+
class Fetch(object):
"""Base class for 'fetch'ing data"""
def __init__(self, urls = []):
self.urls = []
- for url in urls:
- if self.supports(bb.decodeurl(url), d) is 1:
- self.urls.append(url)
- def supports(url, d):
- """Check to see if this fetch class supports a given url.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, urldata, d):
+ """
+ Check to see if this fetch class supports a given url.
"""
return 0
- supports = staticmethod(supports)
- def localpath(url, d):
- """Return the local filename of a given url assuming a successful fetch.
+ def localpath(self, url, urldata, d):
+ """
+ Return the local filename of a given url assuming a successful fetch.
+ Can also setup variables in urldata for use in go (saving code duplication
+ and duplicate code execution)
"""
return url
- localpath = staticmethod(localpath)
def setUrls(self, urls):
self.__urls = urls
@@ -137,16 +172,17 @@ class Fetch(object):
urls = property(getUrls, setUrls, None, "Urls property")
- def setData(self, data):
- self.__data = data
-
- def getData(self):
- return self.__data
-
- data = property(getData, setData, None, "Data property")
+ def forcefetch(self, url, urldata, d):
+ """
+ Force a fetch, even if localpath exists?
+ """
+ return False
- def go(self, urls = []):
- """Fetch urls"""
+ def go(self, url, urldata, d):
+ """
+ Fetch urls
+ Assumes localpath was called first
+ """
raise NoMethodError("Missing implementation for url")
def getSRCDate(d):
@@ -155,7 +191,12 @@ class Fetch(object):
d the bb.data module
"""
- return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 )
+ pn = data.getVar("PN", d, 1)
+
+ if pn:
+ return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("DATE", d, 1)
+
+ return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
getSRCDate = staticmethod(getSRCDate)
def try_mirror(d, tarfn):
@@ -168,6 +209,11 @@ class Fetch(object):
d Is a bb.data instance
tarfn is the name of the tarball
"""
+ tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
+ if os.access(tarpath, os.R_OK):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
+ return True
+
pn = data.getVar('PN', d, True)
src_tarball_stash = None
if pn:
@@ -176,36 +222,45 @@ class Fetch(object):
for stash in src_tarball_stash:
fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
uri = stash + tarfn
- bb.note("fetch " + uri)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
ret = os.system(fetchcmd)
if ret == 0:
- bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn)
return True
return False
try_mirror = staticmethod(try_mirror)
- def check_for_tarball(d, tarfn, dldir, date):
+ def verify_md5sum(ud, got_sum):
"""
- Check for a local copy then check the tarball stash.
- Both checks are skipped if date == 'now'.
-
- d Is a bb.data instance
- tarfn is the name of the tarball
- date is the SRCDATE
+ Verify the md5sum we wanted with the one we got
"""
- if "now" != date:
- dl = os.path.join(dldir, tarfn)
- if os.access(dl, os.R_OK):
- bb.debug(1, "%s already exists, skipping checkout." % tarfn)
- return True
-
- # try to use the tarball stash
- if Fetch.try_mirror(d, tarfn):
- return True
- return False
- check_for_tarball = staticmethod(check_for_tarball)
-
+ wanted_sum = None
+ if 'md5sum' in ud.parm:
+ wanted_sum = ud.parm['md5sum']
+ if not wanted_sum:
+ return True
+
+ return wanted_sum == got_sum
+ verify_md5sum = staticmethod(verify_md5sum)
+
+ def write_md5sum(url, ud, d):
+ if bb.which(data.getVar('PATH', d), 'md5sum'):
+ try:
+ md5pipe = os.popen('md5sum ' + ud.localpath)
+ md5data = (md5pipe.readline().split() or [ "" ])[0]
+ md5pipe.close()
+ except OSError:
+ md5data = ""
+
+ # verify the md5sum
+ if not Fetch.verify_md5sum(ud, md5data):
+ raise MD5SumError(url)
+
+ md5out = file(ud.md5, 'w')
+ md5out.write(md5data)
+ md5out.close()
+ write_md5sum = staticmethod(write_md5sum)
import cvs
import git
@@ -214,6 +269,7 @@ import svn
import wget
import svk
import ssh
+import perforce
methods.append(cvs.Cvs())
methods.append(git.Git())
@@ -222,3 +278,4 @@ methods.append(svn.Svn())
methods.append(wget.Wget())
methods.append(svk.Svk())
methods.append(ssh.SSH())
+methods.append(perforce.Perforce())
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 0b2477560..3bdac177e 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -33,164 +33,119 @@ from bb.fetch import FetchError
from bb.fetch import MissingParameterError
class Cvs(Fetch):
- """Class to fetch a module or modules from cvs repositories"""
- def supports(url, d):
- """Check to see if a given url can be fetched with cvs.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ """
+ Class to fetch a module or modules from cvs repositories
+ """
+ def supports(self, url, ud, d):
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['cvs', 'pserver']
- supports = staticmethod(supports)
-
- def localpath(url, d):
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- if "localpath" in parm:
-# if user overrides local path, use it.
- return parm["localpath"]
+ Check to see if a given url can be fetched with cvs.
+ """
+ return ud.type in ['cvs', 'pserver']
- if not "module" in parm:
+ def localpath(self, url, ud, d):
+ if not "module" in ud.parm:
raise MissingParameterError("cvs method needs a 'module' parameter")
- else:
- module = parm["module"]
- if 'tag' in parm:
- tag = parm['tag']
- else:
- tag = ""
- if 'date' in parm:
- date = parm['date']
- else:
- if not tag:
- date = Fetch.getSRCDate(d)
- else:
- date = ""
+ ud.module = ud.parm["module"]
+
+ ud.tag = ""
+ if 'tag' in ud.parm:
+ ud.tag = ud.parm['tag']
+
+ # Override the default date in certain cases
+ if 'date' in ud.parm:
+ ud.date = ud.parm['date']
+ elif ud.tag:
+ ud.date = ""
+
+ ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date), d)
+
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d))
- localpath = staticmethod(localpath)
+ def forcefetch(self, url, ud, d):
+ if (ud.date == "now"):
+ return True
+ return False
- def go(self, d, urls = []):
- """Fetch urls"""
- if not urls:
- urls = self.urls
+ def go(self, loc, ud, d):
+
+ # try to use the tarball stash
+ if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
+ return
+
+ method = "pserver"
+ if "method" in ud.parm:
+ method = ud.parm["method"]
+
+ localdir = ud.module
+ if "localdir" in ud.parm:
+ localdir = ud.parm["localdir"]
+
+ cvs_rsh = None
+ if method == "ext":
+ if "rsh" in ud.parm:
+ cvs_rsh = ud.parm["rsh"]
+
+ if method == "dir":
+ cvsroot = ud.path
+ else:
+ cvsroot = ":" + method + ":" + ud.user
+ if ud.pswd:
+ cvsroot += ":" + ud.pswd
+ cvsroot += "@" + ud.host + ":" + ud.path
+
+ options = []
+ if ud.date:
+ options.append("-D %s" % ud.date)
+ if ud.tag:
+ options.append("-r %s" % ud.tag)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
- for loc in urls:
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
- if not "module" in parm:
- raise MissingParameterError("cvs method needs a 'module' parameter")
- else:
- module = parm["module"]
-
- dlfile = self.localpath(loc, localdata)
- dldir = data.getVar('DL_DIR', localdata, 1)
-# if local path contains the cvs
-# module, consider the dir above it to be the
-# download directory
-# pos = dlfile.find(module)
-# if pos:
-# dldir = dlfile[:pos]
-# else:
-# dldir = os.path.dirname(dlfile)
-
-# setup cvs options
- options = []
- if 'tag' in parm:
- tag = parm['tag']
- else:
- tag = ""
-
- if 'date' in parm:
- date = parm['date']
- else:
- if not tag:
- date = Fetch.getSRCDate(d)
- else:
- date = ""
-
- if "method" in parm:
- method = parm["method"]
- else:
- method = "pserver"
-
- if "localdir" in parm:
- localdir = parm["localdir"]
- else:
- localdir = module
-
- cvs_rsh = None
- if method == "ext":
- if "rsh" in parm:
- cvs_rsh = parm["rsh"]
-
- tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
- data.setVar('TARFILES', dlfile, localdata)
- data.setVar('TARFN', tarfn, localdata)
-
- if Fetch.check_for_tarball(d, tarfn, dldir, date):
- continue
-
- if date:
- options.append("-D %s" % date)
- if tag:
- options.append("-r %s" % tag)
-
- olddir = os.path.abspath(os.getcwd())
- os.chdir(data.expand(dldir, localdata))
-
-# setup cvsroot
- if method == "dir":
- cvsroot = path
- else:
- cvsroot = ":" + method + ":" + user
- if pswd:
- cvsroot += ":" + pswd
- cvsroot += "@" + host + ":" + path
-
- data.setVar('CVSROOT', cvsroot, localdata)
- data.setVar('CVSCOOPTS', " ".join(options), localdata)
- data.setVar('CVSMODULE', module, localdata)
- cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
- cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
-
- if cvs_rsh:
- cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
- cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
-
-# create module directory
- bb.debug(2, "Fetch: checking for module directory")
- pkg=data.expand('${PN}', d)
- pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg)
- moddir=os.path.join(pkgdir,localdir)
- if os.access(os.path.join(moddir,'CVS'), os.R_OK):
- bb.note("Update " + loc)
-# update sources there
- os.chdir(moddir)
- myret = os.system(cvsupdatecmd)
- else:
- bb.note("Fetch " + loc)
-# check out sources there
- bb.mkdirhier(pkgdir)
- os.chdir(pkgdir)
- bb.debug(1, "Running %s" % cvscmd)
- myret = os.system(cvscmd)
-
- if myret != 0 or not os.access(moddir, os.R_OK):
- try:
- os.rmdir(moddir)
- except OSError:
- pass
- raise FetchError(module)
-
+ data.setVar('CVSROOT', cvsroot, localdata)
+ data.setVar('CVSCOOPTS', " ".join(options), localdata)
+ data.setVar('CVSMODULE', ud.module, localdata)
+ cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
+ cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
+
+ if cvs_rsh:
+ cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
+ cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
+
+ # create module directory
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory")
+ pkg = data.expand('${PN}', d)
+ pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
+ moddir = os.path.join(pkgdir,localdir)
+ if os.access(os.path.join(moddir,'CVS'), os.R_OK):
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
+ # update sources there
os.chdir(moddir)
- os.chdir('..')
-# tar them up to a defined filename
- myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir)))
- if myret != 0:
- try:
- os.unlink(tarfn)
- except OSError:
- pass
- os.chdir(olddir)
- del localdata
+ myret = os.system(cvsupdatecmd)
+ else:
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
+ # check out sources there
+ bb.mkdirhier(pkgdir)
+ os.chdir(pkgdir)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cvscmd)
+ myret = os.system(cvscmd)
+
+ if myret != 0 or not os.access(moddir, os.R_OK):
+ try:
+ os.rmdir(moddir)
+ except OSError:
+ pass
+ raise FetchError(ud.module)
+
+ os.chdir(moddir)
+ os.chdir('..')
+ # tar them up to a defined filename
+ myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(moddir)))
+ if myret != 0:
+ try:
+ os.unlink(ud.localpath)
+ except OSError:
+ pass
+ raise FetchError(ud.module)
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 49235c141..75a762922 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -37,7 +37,7 @@ def prunedir(topdir):
def rungitcmd(cmd,d):
- bb.debug(1, "Running %s" % cmd)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd)
# Need to export PATH as git is likely to be in metadata paths
# rather than host provided
@@ -48,108 +48,80 @@ def rungitcmd(cmd,d):
if myret != 0:
raise FetchError("Git: %s failed" % pathcmd)
-def gettag(parm):
- if 'tag' in parm:
- tag = parm['tag']
- else:
- tag = ""
- if not tag:
- tag = "master"
-
- return tag
-
-def getprotocol(parm):
- if 'protocol' in parm:
- proto = parm['protocol']
- else:
- proto = ""
- if not proto:
- proto = "rsync"
-
- return proto
-
-def localfile(url, d):
- """Return the filename to cache the checkout in"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-
- #if user sets localpath for file, use it instead.
- if "localpath" in parm:
- return parm["localpath"]
-
- tag = gettag(parm)
-
- return data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d)
-
class Git(Fetch):
"""Class to fetch a module or modules from git repositories"""
- def supports(url, d):
- """Check to see if a given url can be fetched with cvs.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, ud, d):
+ """
+ Check to see if a given url can be fetched with cvs.
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['git']
- supports = staticmethod(supports)
+ return ud.type in ['git']
- def localpath(url, d):
+ def localpath(self, url, ud, d):
- return os.path.join(data.getVar("DL_DIR", d, 1), localfile(url, d))
+ ud.proto = "rsync"
+ if 'protocol' in ud.parm:
+ ud.proto = ud.parm['protocol']
- localpath = staticmethod(localpath)
+ ud.tag = "master"
+ if 'tag' in ud.parm:
+ ud.tag = ud.parm['tag']
- def go(self, d, urls = []):
- """Fetch urls"""
- if not urls:
- urls = self.urls
+ ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d)
- for loc in urls:
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- tag = gettag(parm)
- proto = getprotocol(parm)
+ def forcefetch(self, url, ud, d):
+ # tag=="master" must always update
+ if (ud.tag == "master"):
+ return True
+ return False
- gitsrcname = '%s%s' % (host, path.replace('/', '.'))
+ def go(self, loc, ud, d):
+ """Fetch url"""
- repofilename = 'git_%s.tar.gz' % (gitsrcname)
- repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
- repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
+ if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
+ return
- coname = '%s' % (tag)
- codir = os.path.join(repodir, coname)
+ gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
- cofile = self.localpath(loc, d)
+ repofilename = 'git_%s.tar.gz' % (gitsrcname)
+ repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
+ repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
- # tag=="master" must always update
- if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)):
- bb.debug(1, "%s already exists (or was stashed). Skipping git checkout." % cofile)
- continue
+ coname = '%s' % (ud.tag)
+ codir = os.path.join(repodir, coname)
- if not os.path.exists(repodir):
- if Fetch.try_mirror(d, repofilename):
- bb.mkdirhier(repodir)
- os.chdir(repodir)
- rungitcmd("tar -xzf %s" % (repofile),d)
- else:
- rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d)
+ if not os.path.exists(repodir):
+ if Fetch.try_mirror(d, repofilename):
+ bb.mkdirhier(repodir)
+ os.chdir(repodir)
+ rungitcmd("tar -xzf %s" % (repofile),d)
+ else:
+ rungitcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir),d)
- os.chdir(repodir)
- rungitcmd("git pull %s://%s%s" % (proto, host, path),d)
- rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d)
- rungitcmd("git prune-packed", d)
- # old method of downloading tags
- #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
+ os.chdir(repodir)
+ rungitcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path),d)
+ rungitcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path),d)
+ rungitcmd("git prune-packed", d)
+ rungitcmd("git pack-redundant --all | xargs -r rm", d)
+ # Remove all but the .git directory
+ rungitcmd("rm * -Rf", d)
+ # old method of downloading tags
+ #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")),d)
- os.chdir(repodir)
- bb.note("Creating tarball of git repository")
- rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
+ os.chdir(repodir)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
+ rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
- if os.path.exists(codir):
- prunedir(codir)
+ if os.path.exists(codir):
+ prunedir(codir)
- bb.mkdirhier(codir)
- os.chdir(repodir)
- rungitcmd("git read-tree %s" % (tag),d)
- rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
+ bb.mkdirhier(codir)
+ os.chdir(repodir)
+ rungitcmd("git read-tree %s" % (ud.tag),d)
+ rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
- os.chdir(codir)
- bb.note("Creating tarball of git checkout")
- rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
+ os.chdir(codir)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
+ rungitcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ),d)
diff --git a/bitbake/lib/bb/fetch/local.py b/bitbake/lib/bb/fetch/local.py
index 51938f823..522497670 100644
--- a/bitbake/lib/bb/fetch/local.py
+++ b/bitbake/lib/bb/fetch/local.py
@@ -31,15 +31,13 @@ from bb import data
from bb.fetch import Fetch
class Local(Fetch):
- def supports(url, d):
- """Check to see if a given url can be fetched in the local filesystem.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, urldata, d):
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['file','patch']
- supports = staticmethod(supports)
+ Check to see if a given url can be fetched with cvs.
+ """
+ return urldata.type in ['file','patch']
- def localpath(url, d):
+ def localpath(self, url, urldata, d):
"""Return the local filename of a given url assuming a successful fetch.
"""
path = url.split("://")[1]
@@ -52,10 +50,10 @@ class Local(Fetch):
filesdir = data.getVar('FILESDIR', d, 1)
if filesdir:
newpath = os.path.join(filesdir, path)
+ # We don't set localfile as for this fetcher the file is already local!
return newpath
- localpath = staticmethod(localpath)
- def go(self, urls = []):
+ def go(self, url, urldata, d):
"""Fetch urls (no-op for Local method)"""
-# no need to fetch local files, we'll deal with them in place.
+ # no need to fetch local files, we'll deal with them in place.
return 1
diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py
new file mode 100644
index 000000000..88acf6995
--- /dev/null
+++ b/bitbake/lib/bb/fetch/perforce.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004 Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA.
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from bb import data
+from bb.fetch import Fetch
+from bb.fetch import FetchError
+from bb.fetch import MissingParameterError
+
+class Perforce(Fetch):
+ def supports(self, url, ud, d):
+ return ud.type in ['p4']
+
+ def doparse(url,d):
+ parm=[]
+ path = url.split("://")[1]
+ delim = path.find("@");
+ if delim != -1:
+ (user,pswd,host,port) = path.split('@')[0].split(":")
+ path = path.split('@')[1]
+ else:
+ (host,port) = data.getVar('P4PORT', d).split(':')
+ user = ""
+ pswd = ""
+
+ if path.find(";") != -1:
+ keys=[]
+ values=[]
+ plist = path.split(';')
+ for item in plist:
+ if item.count('='):
+ (key,value) = item.split('=')
+ keys.append(key)
+ values.append(value)
+
+ parm = dict(zip(keys,values))
+ path = "//" + path.split(';')[0]
+ host += ":%s" % (port)
+ parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
+
+ return host,path,user,pswd,parm
+ doparse = staticmethod(doparse)
+
+ def getcset(d, depot,host,user,pswd,parm):
+ if "cset" in parm:
+ return parm["cset"];
+ if user:
+ data.setVar('P4USER', user, d)
+ if pswd:
+ data.setVar('P4PASSWD', pswd, d)
+ if host:
+ data.setVar('P4PORT', host, d)
+
+ p4date = data.getVar("P4DATE", d, 1)
+ if "revision" in parm:
+ depot += "#%s" % (parm["revision"])
+ elif "label" in parm:
+ depot += "@%s" % (parm["label"])
+ elif p4date:
+ depot += "@%s" % (p4date)
+
+ p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s changes -m 1 %s" % (p4cmd, depot))
+ p4file = os.popen("%s changes -m 1 %s" % (p4cmd,depot))
+ cset = p4file.readline().strip()
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "READ %s" % (cset))
+ if not cset:
+ return -1
+
+ return cset.split(' ')[1]
+ getcset = staticmethod(getcset)
+
+ def localpath(self, url, ud, d):
+
+ (host,path,user,pswd,parm) = Perforce.doparse(url,d)
+
+ # If a label is specified, we use that as our filename
+
+ if "label" in parm:
+ ud.localfile = "%s.tar.gz" % (parm["label"])
+ return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
+
+ base = path
+ which = path.find('/...')
+ if which != -1:
+ base = path[:which]
+
+ if base[0] == "/":
+ base = base[1:]
+
+ cset = Perforce.getcset(d, path, host, user, pswd, parm)
+
+ ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host,base.replace('/', '.'), cset), d)
+
+ return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
+
+ def go(self, loc, ud, d):
+ """
+ Fetch urls
+ """
+
+ # try to use the tarball stash
+ if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
+ return
+
+ (host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
+
+ if depot.find('/...') != -1:
+ path = depot[:depot.find('/...')]
+ else:
+ path = depot
+
+ if "module" in parm:
+ module = parm["module"]
+ else:
+ module = os.path.basename(path)
+
+ localdata = data.createCopy(d)
+ data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata)
+ data.update_data(localdata)
+
+ # Get the p4 command
+ if user:
+ data.setVar('P4USER', user, localdata)
+
+ if pswd:
+ data.setVar('P4PASSWD', pswd, localdata)
+
+ if host:
+ data.setVar('P4PORT', host, localdata)
+
+ p4cmd = data.getVar('FETCHCOMMAND', localdata, 1)
+
+ # create temp directory
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
+ bb.mkdirhier(data.expand('${WORKDIR}', localdata))
+ data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata)
+ tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
+ tmpfile = tmppipe.readline().strip()
+ if not tmpfile:
+ bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+ raise FetchError(module)
+
+ if "label" in parm:
+ depot = "%s@%s" % (depot,parm["label"])
+ else:
+ cset = Perforce.getcset(d, depot, host, user, pswd, parm)
+ depot = "%s@%s" % (depot,cset)
+
+ os.chdir(tmpfile)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "%s files %s" % (p4cmd, depot))
+ p4file = os.popen("%s files %s" % (p4cmd, depot))
+
+ if not p4file:
+ bb.error("Fetch: unable to get the P4 files from %s" % (depot))
+ raise FetchError(module)
+
+ count = 0
+
+ for file in p4file:
+ list = file.split()
+
+ if list[2] == "delete":
+ continue
+
+ dest = list[0][len(path)+1:]
+ where = dest.find("#")
+
+ os.system("%s print -o %s/%s %s" % (p4cmd, module,dest[:where],list[0]))
+ count = count + 1
+
+ if count == 0:
+ bb.error("Fetch: No files gathered from the P4 fetch")
+ raise FetchError(module)
+
+ myret = os.system("tar -czf %s %s" % (ud.localpath, module))
+ if myret != 0:
+ try:
+ os.unlink(ud.localpath)
+ except OSError:
+ pass
+ raise FetchError(module)
+ # cleanup
+ os.system('rm -rf %s' % tmpfile)
+
+
diff --git a/bitbake/lib/bb/fetch/ssh.py b/bitbake/lib/bb/fetch/ssh.py
index 57874d5ba..e5f69e33e 100644
--- a/bitbake/lib/bb/fetch/ssh.py
+++ b/bitbake/lib/bb/fetch/ssh.py
@@ -64,59 +64,55 @@ __pattern__ = re.compile(r'''
class SSH(Fetch):
'''Class to fetch a module or modules via Secure Shell'''
- def supports(self, url, d):
+ def supports(self, url, urldata, d):
return __pattern__.match(url) != None
- def localpath(self, url, d):
+ def localpath(self, url, urldata, d):
m = __pattern__.match(url)
path = m.group('path')
host = m.group('host')
- lpath = os.path.join(data.getVar('DL_DIR', d, 1), host, os.path.basename(path))
+ lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
return lpath
- def go(self, d, urls = []):
- if not urls:
- urls = self.urls
-
- for url in urls:
- dldir = data.getVar('DL_DIR', d, 1)
-
- m = __pattern__.match(url)
- path = m.group('path')
- host = m.group('host')
- port = m.group('port')
- user = m.group('user')
- password = m.group('pass')
-
- ldir = os.path.join(dldir, host)
- lpath = os.path.join(ldir, os.path.basename(path))
-
- if not os.path.exists(ldir):
- os.makedirs(ldir)
-
- if port:
- port = '-P %s' % port
- else:
- port = ''
-
- if user:
- fr = user
- if password:
- fr += ':%s' % password
- fr += '@%s' % host
- else:
- fr = host
- fr += ':%s' % path
-
-
- import commands
- cmd = 'scp -B -r %s %s %s/' % (
- port,
- commands.mkarg(fr),
- commands.mkarg(ldir)
- )
-
- (exitstatus, output) = commands.getstatusoutput(cmd)
- if exitstatus != 0:
- print output
- raise FetchError('Unable to fetch %s' % url)
+ def go(self, url, urldata, d):
+ dldir = data.getVar('DL_DIR', d, 1)
+
+ m = __pattern__.match(url)
+ path = m.group('path')
+ host = m.group('host')
+ port = m.group('port')
+ user = m.group('user')
+ password = m.group('pass')
+
+ ldir = os.path.join(dldir, host)
+ lpath = os.path.join(ldir, os.path.basename(path))
+
+ if not os.path.exists(ldir):
+ os.makedirs(ldir)
+
+ if port:
+ port = '-P %s' % port
+ else:
+ port = ''
+
+ if user:
+ fr = user
+ if password:
+ fr += ':%s' % password
+ fr += '@%s' % host
+ else:
+ fr = host
+ fr += ':%s' % path
+
+
+ import commands
+ cmd = 'scp -B -r %s %s %s/' % (
+ port,
+ commands.mkarg(fr),
+ commands.mkarg(ldir)
+ )
+
+ (exitstatus, output) = commands.getstatusoutput(cmd)
+ if exitstatus != 0:
+ print output
+ raise FetchError('Unable to fetch %s' % url)
diff --git a/bitbake/lib/bb/fetch/svk.py b/bitbake/lib/bb/fetch/svk.py
index 19103213c..29270ab3d 100644
--- a/bitbake/lib/bb/fetch/svk.py
+++ b/bitbake/lib/bb/fetch/svk.py
@@ -42,112 +42,76 @@ from bb.fetch import MissingParameterError
class Svk(Fetch):
"""Class to fetch a module or modules from svk repositories"""
- def supports(url, d):
- """Check to see if a given url can be fetched with svk.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, ud, d):
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['svk']
- supports = staticmethod(supports)
-
- def localpath(url, d):
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- if "localpath" in parm:
-# if user overrides local path, use it.
- return parm["localpath"]
+ Check to see if a given url can be fetched with cvs.
+ """
+ return ud.type in ['svk']
- if not "module" in parm:
+ def localpath(self, url, ud, d):
+ if not "module" in ud.parm:
raise MissingParameterError("svk method needs a 'module' parameter")
else:
- module = parm["module"]
- if 'rev' in parm:
- revision = parm['rev']
- else:
- revision = ""
+ ud.module = ud.parm["module"]
+
+ ud.revision = ""
+ if 'rev' in ud.parm:
+ ud.revision = ud.parm['rev']
+
+ ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
- date = Fetch.getSRCDate(d)
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, path.replace('/', '.'), revision, date), d))
- localpath = staticmethod(localpath)
+ def forcefetch(self, url, ud, d):
+ if (ud.date == "now"):
+ return True
+ return False
- def go(self, d, urls = []):
+ def go(self, loc, ud, d):
"""Fetch urls"""
- if not urls:
- urls = self.urls
+ if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
+ return
+
+ svkroot = ud.host + ud.path
+
+ svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module)
+
+ if ud.revision:
+ svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module)
+
+ # create temp directory
localdata = data.createCopy(d)
- data.setVar('OVERRIDES', "svk:%s" % data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
-
- for loc in urls:
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
- if not "module" in parm:
- raise MissingParameterError("svk method needs a 'module' parameter")
- else:
- module = parm["module"]
-
- dlfile = self.localpath(loc, localdata)
- dldir = data.getVar('DL_DIR', localdata, 1)
-
-# setup svk options
- options = []
- if 'rev' in parm:
- revision = parm['rev']
- else:
- revision = ""
-
- date = Fetch.getSRCDate(d)
- tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata)
- data.setVar('TARFILES', dlfile, localdata)
- data.setVar('TARFN', tarfn, localdata)
-
- if Fetch.check_for_tarball(d, tarfn, dldir, date):
- continue
-
- olddir = os.path.abspath(os.getcwd())
- os.chdir(data.expand(dldir, localdata))
-
- svkroot = host + path
-
- data.setVar('SVKROOT', svkroot, localdata)
- data.setVar('SVKCOOPTS', " ".join(options), localdata)
- data.setVar('SVKMODULE', module, localdata)
- svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, module)
-
- if revision:
- svkcmd = "svk co -r %s/%s" % (revision, svkroot, module)
-
-# create temp directory
- bb.debug(2, "Fetch: creating temporary directory")
- bb.mkdirhier(data.expand('${WORKDIR}', localdata))
- data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
- tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
- tmpfile = tmppipe.readline().strip()
- if not tmpfile:
- bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
- raise FetchError(module)
-
-# check out sources there
- os.chdir(tmpfile)
- bb.note("Fetch " + loc)
- bb.debug(1, "Running %s" % svkcmd)
- myret = os.system(svkcmd)
- if myret != 0:
- try:
- os.rmdir(tmpfile)
- except OSError:
- pass
- raise FetchError(module)
-
- os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
-# tar them up to a defined filename
- myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
- if myret != 0:
- try:
- os.unlink(tarfn)
- except OSError:
- pass
-# cleanup
- os.system('rm -rf %s' % tmpfile)
- os.chdir(olddir)
- del localdata
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
+ bb.mkdirhier(data.expand('${WORKDIR}', localdata))
+ data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
+ tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
+ tmpfile = tmppipe.readline().strip()
+ if not tmpfile:
+ bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+ raise FetchError(ud.module)
+
+ # check out sources there
+ os.chdir(tmpfile)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd)
+ myret = os.system(svkcmd)
+ if myret != 0:
+ try:
+ os.rmdir(tmpfile)
+ except OSError:
+ pass
+ raise FetchError(ud.module)
+
+ os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
+ # tar them up to a defined filename
+ myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
+ if myret != 0:
+ try:
+ os.unlink(ud.localpath)
+ except OSError:
+ pass
+ raise FetchError(ud.module)
+ # cleanup
+ os.system('rm -rf %s' % tmpfile)
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index d1a959371..b95de2a79 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -26,6 +26,7 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig
"""
import os, re
+import sys
import bb
from bb import data
from bb.fetch import Fetch
@@ -34,136 +35,98 @@ from bb.fetch import MissingParameterError
class Svn(Fetch):
"""Class to fetch a module or modules from svn repositories"""
- def supports(url, d):
- """Check to see if a given url can be fetched with svn.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, ud, d):
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['svn']
- supports = staticmethod(supports)
-
- def localpath(url, d):
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- if "localpath" in parm:
-# if user overrides local path, use it.
- return parm["localpath"]
+ Check to see if a given url can be fetched with svn.
+ """
+ return ud.type in ['svn']
- if not "module" in parm:
+ def localpath(self, url, ud, d):
+ if not "module" in ud.parm:
raise MissingParameterError("svn method needs a 'module' parameter")
else:
- module = parm["module"]
- if 'rev' in parm:
- revision = parm['rev']
- else:
- revision = ""
+ ud.module = ud.parm["module"]
+
+ ud.revision = ""
+ if 'rev' in ud.parm:
+ ud.revision = ud.parm['rev']
+
+ ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
+
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
- date = Fetch.getSRCDate(d)
+ def forcefetch(self, url, ud, d):
+ if (ud.date == "now"):
+ return True
+ return False
- return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, path.replace('/', '.'), revision, date), d))
- localpath = staticmethod(localpath)
+ def go(self, loc, ud, d):
+ """Fetch url"""
- def go(self, d, urls = []):
- """Fetch urls"""
- if not urls:
- urls = self.urls
+ # try to use the tarball stash
+ if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
+ return
+
+ proto = "svn"
+ if "proto" in ud.parm:
+ proto = ud.parm["proto"]
+
+ svn_rsh = None
+ if proto == "svn+ssh" and "rsh" in ud.parm:
+ svn_rsh = ud.parm["rsh"]
+
+ svnroot = ud.host + ud.path
+
+ # either use the revision, or SRCDATE in braces, or nothing for SRCDATE = "now"
+ options = []
+ if ud.revision:
+ options.append("-r %s" % ud.revision)
+ elif ud.date != "now":
+ options.append("-r {%s}" % ud.date)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
- for loc in urls:
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
- if not "module" in parm:
- raise MissingParameterError("svn method needs a 'module' parameter")
- else:
- module = parm["module"]
-
- dlfile = self.localpath(loc, localdata)
- dldir = data.getVar('DL_DIR', localdata, 1)
-# if local path contains the svn
-# module, consider the dir above it to be the
-# download directory
-# pos = dlfile.find(module)
-# if pos:
-# dldir = dlfile[:pos]
-# else:
-# dldir = os.path.dirname(dlfile)
-
-# setup svn options
- options = []
- if 'rev' in parm:
- revision = parm['rev']
- else:
- revision = ""
-
- date = Fetch.getSRCDate(d)
-
- if "proto" in parm:
- proto = parm["proto"]
- else:
- proto = "svn"
-
- svn_rsh = None
- if proto == "svn+ssh" and "rsh" in parm:
- svn_rsh = parm["rsh"]
-
- tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata)
- data.setVar('TARFILES', dlfile, localdata)
- data.setVar('TARFN', tarfn, localdata)
-
- if Fetch.check_for_tarball(d, tarfn, dldir, date):
- continue
-
- olddir = os.path.abspath(os.getcwd())
- os.chdir(data.expand(dldir, localdata))
-
- svnroot = host + path
-
- data.setVar('SVNROOT', svnroot, localdata)
- data.setVar('SVNCOOPTS', " ".join(options), localdata)
- data.setVar('SVNMODULE', module, localdata)
- svncmd = data.getVar('FETCHCOMMAND', localdata, 1)
- svncmd = "svn co -r {%s} %s://%s/%s" % (date, proto, svnroot, module)
-
- if revision:
- svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module)
- elif date == "now":
- svncmd = "svn co %s://%s/%s" % (proto, svnroot, module)
-
- if svn_rsh:
- svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
-
-# create temp directory
- bb.debug(2, "Fetch: creating temporary directory")
- bb.mkdirhier(data.expand('${WORKDIR}', localdata))
- data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata)
- tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
- tmpfile = tmppipe.readline().strip()
- if not tmpfile:
- bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
- raise FetchError(module)
-
-# check out sources there
- os.chdir(tmpfile)
- bb.note("Fetch " + loc)
- bb.debug(1, "Running %s" % svncmd)
+ data.setVar('SVNROOT', "%s://%s/%s" % (proto, svnroot, ud.module), localdata)
+ data.setVar('SVNCOOPTS', " ".join(options), localdata)
+ data.setVar('SVNMODULE', ud.module, localdata)
+ svncmd = data.getVar('FETCHCOMMAND', localdata, 1)
+ svnupcmd = data.getVar('UPDATECOMMAND', localdata, 1)
+
+ if svn_rsh:
+ svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
+ svnupcmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svnupcmd)
+
+ pkg = data.expand('${PN}', d)
+ pkgdir = os.path.join(data.expand('${SVNDIR}', localdata), pkg)
+ moddir = os.path.join(pkgdir, ud.module)
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + moddir + "'")
+
+ if os.access(os.path.join(moddir, '.svn'), os.R_OK):
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
+ # update sources there
+ os.chdir(moddir)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupcmd)
+ myret = os.system(svnupcmd)
+ else:
+ bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
+ # check out sources there
+ bb.mkdirhier(pkgdir)
+ os.chdir(pkgdir)
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svncmd)
myret = os.system(svncmd)
- if myret != 0:
- try:
- os.rmdir(tmpfile)
- except OSError:
- pass
- raise FetchError(module)
-
- os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
-# tar them up to a defined filename
- myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
- if myret != 0:
- try:
- os.unlink(tarfn)
- except OSError:
- pass
-# cleanup
- os.system('rm -rf %s' % tmpfile)
- os.chdir(olddir)
- del localdata
+
+ if myret != 0:
+ raise FetchError(ud.module)
+
+ os.chdir(pkgdir)
+ # tar them up to a defined filename
+ myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
+ if myret != 0:
+ try:
+ os.unlink(ud.localpath)
+ except OSError:
+ pass
+ raise FetchError(ud.module)
diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py
index e47a8859b..9c9c1675a 100644
--- a/bitbake/lib/bb/fetch/wget.py
+++ b/bitbake/lib/bb/fetch/wget.py
@@ -30,138 +30,70 @@ import bb
from bb import data
from bb.fetch import Fetch
from bb.fetch import FetchError
-from bb.fetch import MD5SumError
from bb.fetch import uri_replace
class Wget(Fetch):
"""Class to fetch urls via 'wget'"""
- def supports(url, d):
- """Check to see if a given url can be fetched using wget.
- Expects supplied url in list form, as outputted by bb.decodeurl().
+ def supports(self, url, ud, d):
"""
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- return type in ['http','https','ftp']
- supports = staticmethod(supports)
-
- def localpath(url, d):
-# strip off parameters
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
- if "localpath" in parm:
-# if user overrides local path, use it.
- return parm["localpath"]
- url = bb.encodeurl([type, host, path, user, pswd, {}])
-
- return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url))
- localpath = staticmethod(localpath)
-
- def go(self, d, urls = []):
+ Check to see if a given url can be fetched with cvs.
+ """
+ return ud.type in ['http','https','ftp']
+
+ def localpath(self, url, ud, d):
+
+ url = bb.encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}])
+ ud.basename = os.path.basename(ud.path)
+ ud.localfile = data.expand(os.path.basename(url), d)
+
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
+
+ def go(self, uri, ud, d):
"""Fetch urls"""
- def md5_sum(parm, d):
- """
- Return the MD5SUM associated with the to be downloaded
- file.
- It can return None if no md5sum is associated
- """
- try:
- return parm['md5sum']
- except:
- return None
-
- def verify_md5sum(wanted_sum, got_sum):
- """
- Verify the md5sum we wanted with the one we got
- """
- if not wanted_sum:
- return True
-
- return wanted_sum == got_sum
-
- def fetch_uri(uri, basename, dl, md5, parm, d):
- # the MD5 sum we want to verify
- wanted_md5sum = md5_sum(parm, d)
- if os.path.exists(dl):
-# file exists, but we didnt complete it.. trying again..
+ def fetch_uri(uri, ud, d):
+ if os.path.exists(ud.localpath):
+ # file exists, but we didnt complete it.. trying again..
fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
else:
fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
- bb.note("fetch " + uri)
+ bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
- fetchcmd = fetchcmd.replace("${FILE}", basename)
- bb.debug(2, "executing " + fetchcmd)
+ fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd)
ret = os.system(fetchcmd)
if ret != 0:
return False
# check if sourceforge did send us to the mirror page
- dl_dir = data.getVar("DL_DIR", d, True)
- if not os.path.exists(dl):
- os.system("rm %s*" % dl) # FIXME shell quote it
- bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename)
+ if not os.path.exists(ud.localpath):
+ os.system("rm %s*" % ud.localpath) # FIXME shell quote it
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "sourceforge.net send us to the mirror on %s" % ud.basename)
return False
-# supposedly complete.. write out md5sum
- if bb.which(data.getVar('PATH', d), 'md5sum'):
- try:
- md5pipe = os.popen('md5sum ' + dl)
- md5data = (md5pipe.readline().split() or [ "" ])[0]
- md5pipe.close()
- except OSError:
- md5data = ""
-
- # verify the md5sum
- if not verify_md5sum(wanted_md5sum, md5data):
- raise MD5SumError(uri)
-
- md5out = file(md5, 'w')
- md5out.write(md5data)
- md5out.close()
return True
- if not urls:
- urls = self.urls
-
localdata = data.createCopy(d)
data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
- for uri in urls:
- completed = 0
- (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata))
- basename = os.path.basename(path)
- dl = self.localpath(uri, d)
- dl = data.expand(dl, localdata)
- md5 = dl + '.md5'
-
- if os.path.exists(md5):
-# complete, nothing to see here..
- continue
-
- premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
- for (find, replace) in premirrors:
- newuri = uri_replace(uri, find, replace, d)
- if newuri != uri:
- if fetch_uri(newuri, basename, dl, md5, parm, localdata):
- completed = 1
- break
-
- if completed:
- continue
-
- if fetch_uri(uri, basename, dl, md5, parm, localdata):
- continue
-
-# try mirrors
- mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
- for (find, replace) in mirrors:
- newuri = uri_replace(uri, find, replace, d)
- if newuri != uri:
- if fetch_uri(newuri, basename, dl, md5, parm, localdata):
- completed = 1
- break
-
- if not completed:
- raise FetchError(uri)
-
- del localdata
+ premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
+ for (find, replace) in premirrors:
+ newuri = uri_replace(uri, find, replace, d)
+ if newuri != uri:
+ if fetch_uri(newuri, ud, localdata):
+ return
+
+ if fetch_uri(uri, ud, localdata):
+ return
+
+ # try mirrors
+ mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
+ for (find, replace) in mirrors:
+ newuri = uri_replace(uri, find, replace, d)
+ if newuri != uri:
+ if fetch_uri(newuri, ud, localdata):
+ return
+
+ raise FetchError(uri)
diff --git a/bitbake/lib/bb/methodpool.py b/bitbake/lib/bb/methodpool.py
index d7434ed33..e14986bc1 100644
--- a/bitbake/lib/bb/methodpool.py
+++ b/bitbake/lib/bb/methodpool.py
@@ -61,9 +61,6 @@ def insert_method(modulename, code, fn):
comp = better_compile(code, "<bb>", fn )
better_exec(comp, __builtins__, code, fn)
- # hack hack hack XXX
- return
-
# now some instrumentation
code = comp.co_names
for name in code:
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
new file mode 100644
index 000000000..473851cc7
--- /dev/null
+++ b/bitbake/lib/bb/msg.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'msg' implementation
+
+Message handling infrastructure for bitbake
+
+# Copyright (C) 2006 Richard Purdie
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA.
+
+"""
+
+import sys, os, re, bb
+from bb import utils
+
+debug_level = {}
+
+verbose = False
+
+domain = bb.utils.Enum(
+ 'Build',
+ 'Cache',
+ 'Collection',
+ 'Data',
+ 'Depends',
+ 'Fetcher',
+ 'Parsing',
+ 'Provider',
+ 'RunQueue',
+ 'TaskData',
+ 'Util')
+
+#
+# Message control functions
+#
+
+def set_debug_level(level):
+ bb.msg.debug_level = {}
+ for domain in bb.msg.domain:
+ bb.msg.debug_level[domain] = level
+ bb.msg.debug_level['default'] = level
+
+def set_verbose(level):
+ bb.msg.verbose = level
+
+def set_debug_domains(domains):
+ for domain in domains:
+ found = False
+ for ddomain in bb.msg.domain:
+ if domain == str(ddomain):
+ bb.msg.debug_level[ddomain] = bb.msg.debug_level[ddomain] + 1
+ found = True
+ if not found:
+ std_warn("Logging domain %s is not valid, ignoring" % domain)
+
+#
+# Message handling functions
+#
+
+def debug(level, domain, msg, fn = None):
+ if debug_level[domain] >= level:
+ print 'DEBUG: ' + msg
+
+def note(level, domain, msg, fn = None):
+ if level == 1 or verbose or debug_level[domain] >= 1:
+ std_note(msg)
+
+def warn(domain, msg, fn = None):
+ std_warn(msg)
+
+def error(domain, msg, fn = None):
+ std_error(msg)
+
+def fatal(domain, msg, fn = None):
+ std_fatal(msg)
+
+#
+# Compatibility functions for the original message interface
+#
+def std_debug(lvl, msg):
+ if debug_level['default'] >= lvl:
+ print 'DEBUG: ' + msg
+
+def std_note(msg):
+ print 'NOTE: ' + msg
+
+def std_warn(msg):
+ print 'WARNING: ' + msg
+
+def std_error(msg):
+ print 'ERROR: ' + msg
+
+def std_fatal(msg):
+ print 'ERROR: ' + msg
+ sys.exit(1)
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index 58e17d154..70fdba03b 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -37,11 +37,16 @@ class SkipPackage(Exception):
__mtime_cache = {}
def cached_mtime(f):
if not __mtime_cache.has_key(f):
- update_mtime(f)
+ __mtime_cache[f] = os.stat(f)[8]
return __mtime_cache[f]
-def update_mtime(f):
- __mtime_cache[f] = os.stat(f)[8]
+def cached_mtime_noerror(f):
+ if not __mtime_cache.has_key(f):
+ try:
+ __mtime_cache[f] = os.stat(f)[8]
+ except OSError:
+ return 0
+ return __mtime_cache[f]
def mark_dependency(d, f):
if f.startswith('./'):
diff --git a/bitbake/lib/bb/parse/parse_c/BBHandler.py b/bitbake/lib/bb/parse/parse_c/BBHandler.py
index d9f48db17..b430e1f4e 100644
--- a/bitbake/lib/bb/parse/parse_c/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_c/BBHandler.py
@@ -5,33 +5,33 @@
Reads a .bb file and obtains its metadata (using a C++ parser)
Copyright (C) 2006 Tim Robert Ansell
- Copyright (C) 2006 Holger Hans Peter Freyther
-
+ Copyright (C) 2006 Holger Hans Peter Freyther
+
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in all
- copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
- SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
- DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
- THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+ SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
-import os
+import os, sys
# The Module we will use here
import bb
@@ -61,51 +61,126 @@ def supports(fn, data):
return fn[-3:] == ".bb" or fn[-8:] == ".bbclass" or fn[-4:] == ".inc" or fn[-5:] == ".conf"
def init(fn, data):
- if not data.getVar('TOPDIR'):
- bb.error('TOPDIR is not set')
- if not data.getVar('BBPATH'):
- bb.error('BBPATH is not set')
+ if not bb.data.getVar('TOPDIR', data):
+ bb.data.setVar('TOPDIR', os.getcwd(), data)
+ if not bb.data.getVar('BBPATH', data):
+ bb.data.setVar('BBPATH', os.path.join(sys.prefix, 'share', 'bitbake'), data)
+
+def handle_inherit(d):
+ """
+ Handle inheriting of classes. This will load all default classes.
+ It could be faster, it could detect infinite loops but this is todo
+ Also this delayed loading of bb.parse could impose a penalty
+ """
+ from bb.parse import handle
+
+ files = (data.getVar('INHERIT', d, True) or "").split()
+ if not "base" in i:
+ files[0:0] = ["base"]
+
+ __inherit_cache = data.getVar('__inherit_cache', d) or []
+ for f in files:
+ file = data.expand(f, d)
+ if file[0] != "/" and file[-8:] != ".bbclass":
+ file = os.path.join('classes', '%s.bbclass' % file)
+
+ if not file in __inherit_cache:
+ debug(2, "BB %s:%d: inheriting %s" % (fn, lineno, file))
+ __inherit_cache.append( file )
+
+ try:
+ handle(file, d, True)
+ except IOError:
+ print "Failed to inherit %s" % file
+ data.setVar('__inherit_cache', __inherit_cache, d)
def handle(fn, d, include):
- print ""
- print "fn: %s" % fn
- print "data: %s" % d
- print dir(d)
- print d.getVar.__doc__
- print "include: %s" % include
+ from bb import data, parse
+
+ (root, ext) = os.path.splitext(os.path.basename(fn))
+ base_name = "%s%s" % (root,ext)
+
+ # initialize with some data
+ init(fn,d)
# check if we include or are the beginning
+ oldfile = None
if include:
- oldfile = d.getVar('FILE')
- else:
- #d.inheritFromOS()
- oldfile = None
+ oldfile = d.getVar('FILE', False)
+ is_conf = False
+ elif ext == ".conf":
+ is_conf = True
+ data.inheritFromOS(d)
# find the file
if not os.path.isabs(fn):
- bb.error("No Absolute FILE name")
- abs_fn = bb.which(d.getVar('BBPATH'), fn)
+ abs_fn = bb.which(d.getVar('BBPATH', True), fn)
else:
abs_fn = fn
# check if the file exists
if not os.path.exists(abs_fn):
- raise IOError("file '%(fn)' not found" % locals() )
+ raise IOError("file '%(fn)s' not found" % locals() )
# now we know the file is around mark it as dep
if include:
parse.mark_dependency(d, abs_fn)
+ # manipulate the bbpath
+ if ext != ".bbclass" and ext != ".conf":
+ old_bb_path = data.getVar('BBPATH', d)
+ data.setVar('BBPATH', os.path.dirname(abs_fn) + (":%s" %old_bb_path) , d)
+
+ # handle INHERITS and base inherit
+ if ext != ".bbclass" and ext != ".conf":
+ data.setVar('FILE', fn, d)
+ handle_interit(d)
+
# now parse this file - by defering it to C++
- parsefile(fn, d)
+ parsefile(abs_fn, d, is_conf)
+
+ # Finish it up
+ if include == 0:
+ data.expandKeys(d)
+ data.update_data(d)
+ #### !!! XXX Finish it up by executing the anonfunc
+
# restore the original FILE
if oldfile:
d.setVar('FILE', oldfile)
+ # restore bbpath
+ if ext != ".bbclass" and ext != ".conf":
+ data.setVar('BBPATH', old_bb_path, d )
+
+
return d
+
+# Needed for BitBake files...
+__pkgsplit_cache__={}
+def vars_from_file(mypkg, d):
+ if not mypkg:
+ return (None, None, None)
+ if mypkg in __pkgsplit_cache__:
+ return __pkgsplit_cache__[mypkg]
+
+ myfile = os.path.splitext(os.path.basename(mypkg))
+ parts = myfile[0].split('_')
+ __pkgsplit_cache__[mypkg] = parts
+ exp = 3 - len(parts)
+ tmplist = []
+ while exp != 0:
+ exp -= 1
+ tmplist.append(None)
+ parts.extend(tmplist)
+ return parts
+
+
+
+
# Inform bitbake that we are a parser
# We need to define all three
from bb.parse import handlers
diff --git a/bitbake/lib/bb/parse/parse_c/Makefile b/bitbake/lib/bb/parse/parse_c/Makefile
index 9eb7ce9d0..77daccb72 100644
--- a/bitbake/lib/bb/parse/parse_c/Makefile
+++ b/bitbake/lib/bb/parse/parse_c/Makefile
@@ -1,6 +1,6 @@
-test: bitbakec.so
- python test.py
+buil: bitbakec.so
+ echo "Done"
bitbakescanner.cc: bitbakescanner.l
flex -t bitbakescanner.l > bitbakescanner.cc
@@ -28,9 +28,9 @@ bitbakec.so: bitbakec.o bitbakeparser.o bitbakescanner.o
g++ -shared -fPIC bitbakeparser.o bitbakescanner.o bitbakec.o -o bitbakec.so
clean:
- rm *.out
- rm *.cc
- rm bitbakec.c
- rm bitbakec-processed.c
- rm *.o
- rm *.so
+ rm -f *.out
+ rm -f *.cc
+ rm -f bitbakec.c
+ rm -f bitbakec-processed.c
+ rm -f *.o
+ rm -f *.so
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakec.pyx b/bitbake/lib/bb/parse/parse_c/bitbakec.pyx
index 362cc2021..c666e9b6b 100644
--- a/bitbake/lib/bb/parse/parse_c/bitbakec.pyx
+++ b/bitbake/lib/bb/parse/parse_c/bitbakec.pyx
@@ -6,96 +6,107 @@ cdef extern from "stdio.h":
FILE *fopen(char*, char*)
int fclose(FILE *fp)
+cdef extern from "string.h":
+ int strlen(char*)
cdef extern from "lexerc.h":
ctypedef struct lex_t:
void* parser
void* scanner
+ char* name
FILE* file
+ int config
void* data
int lineError
int errorParse
- cdef extern void parse(FILE*, object)
+ cdef extern int parse(FILE*, char*, object, int)
-def parsefile(object file, object data):
- print "parsefile: 1", file, data
+def parsefile(object file, object data, object config):
+ #print "parsefile: 1", file, data
# Open the file
cdef FILE* f
f = fopen(file, "r")
- print "parsefile: 2 opening file"
+ #print "parsefile: 2 opening file"
if (f == NULL):
raise IOError("No such file %s." % file)
- print "parsefile: 3 parse"
- parse(f, data)
+ #print "parsefile: 3 parse"
+ parse(f, file, data, config)
# Close the file
- print "parsefile: 4 closing"
fclose(f)
-
+
cdef public void e_assign(lex_t* container, char* key, char* what):
- print "e_assign", key, what
+ #print "e_assign", key, what
+ if what == NULL:
+ print "FUTURE Warning empty string: use \"\""
+ what = ""
+
d = <object>container.data
- d.setVar(key, what)
+ d.setVar(key, what)
cdef public void e_export(lex_t* c, char* what):
- print "e_export", what
+ #print "e_export", what
#exp:
# bb.data.setVarFlag(key, "export", 1, data)
- d = <object>container.data
- d.setVarFlag(key, "export", 1)
+ d = <object>c.data
+ d.setVarFlag(what, "export", 1)
cdef public void e_immediate(lex_t* c, char* key, char* what):
- print "e_immediate", key, what
+ #print "e_immediate", key, what
#colon:
# val = bb.data.expand(groupd["value"], data)
d = <object>c.data
- d.setVar(key, d.expand(what))
+ d.setVar(key, d.expand(what,d))
cdef public void e_cond(lex_t* c, char* key, char* what):
- print "e_cond", key, what
+ #print "e_cond", key, what
#ques:
# val = bb.data.getVar(key, data)
# if val == None:
# val = groupd["value"]
+ if what == NULL:
+ print "FUTURE warning: Use \"\" for", key
+ what = ""
+
d = <object>c.data
- d.setVar(key, (d.getVar(key) or what))
+ d.setVar(key, (d.getVar(key,False) or what))
cdef public void e_prepend(lex_t* c, char* key, char* what):
- print "e_prepend", key, what
+ #print "e_prepend", key, what
#prepend:
# val = "%s %s" % (groupd["value"], (bb.data.getVar(key, data) or ""))
d = <object>c.data
- d.setVar(key, what + " " + (d.getVar(key) or ""))
+ d.setVar(key, what + " " + (d.getVar(key,0) or ""))
cdef public void e_append(lex_t* c, char* key, char* what):
- print "e_append", key, what
+ #print "e_append", key, what
#append:
# val = "%s %s" % ((bb.data.getVar(key, data) or ""), groupd["value"])
d = <object>c.data
- d.setVar(key, (d.getVar(key) or "") + " " + what)
+ d.setVar(key, (d.getVar(key,0) or "") + " " + what)
cdef public void e_precat(lex_t* c, char* key, char* what):
- print "e_precat", key, what
+ #print "e_precat", key, what
#predot:
# val = "%s%s" % (groupd["value"], (bb.data.getVar(key, data) or ""))
d = <object>c.data
- d.setVar(key, what + (d.getVar(key) or ""))
+ d.setVar(key, what + (d.getVar(key,0) or ""))
cdef public void e_postcat(lex_t* c, char* key, char* what):
- print "e_postcat", key, what
+ #print "e_postcat", key, what
#postdot:
# val = "%s%s" % ((bb.data.getVar(key, data) or ""), groupd["value"])
d = <object>c.data
- d.setVar(key, (d.getVar(key) or "") + what)
+ d.setVar(key, (d.getVar(key,0) or "") + what)
-cdef public void e_addtask(lex_t* c, char* name, char* before, char* after):
- print "e_addtask", name, before, after
+cdef public int e_addtask(lex_t* c, char* name, char* before, char* after) except -1:
+ #print "e_addtask", name
# func = m.group("func")
# before = m.group("before")
# after = m.group("after")
@@ -112,69 +123,131 @@ cdef public void e_addtask(lex_t* c, char* name, char* before, char* after):
# # set up things that depend on this func
# data.setVarFlag(var, "postdeps", before.split(), d)
# return
-
- do = "do_%s" % name
+
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No tasks allowed in config files")
+ return -1
+
d = <object>c.data
+ do = "do_%s" % name
d.setVarFlag(do, "task", 1)
- if strlen(before) > 0:
+ if before != NULL and strlen(before) > 0:
+ #print "Before", before
+ d.setVarFlag(do, "postdeps", ("%s" % before).split())
+ if after != NULL and strlen(after) > 0:
+ #print "After", after
d.setVarFlag(do, "deps", ("%s" % after).split())
- if strlen(after) > 0:
- d.setVarFlag(do, "deps", ("%s" % before).split())
+ return 0
-cdef public void e_addhandler(lex_t* c, char* h):
- print "e_addhandler", h
+cdef public int e_addhandler(lex_t* c, char* h) except -1:
+ #print "e_addhandler", h
# data.setVarFlag(h, "handler", 1, d)
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No handlers allowed in config files")
+ return -1
+
d = <object>c.data
d.setVarFlag(h, "handler", 1)
+ return 0
+
+cdef public int e_export_func(lex_t* c, char* function) except -1:
+ #print "e_export_func", function
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No functions allowed in config files")
+ return -1
+
+ return 0
+
+cdef public int e_inherit(lex_t* c, char* file) except -1:
+ #print "e_inherit", file
-cdef public void e_export_func(lex_t* c, char* function):
- print "e_export_func", function
- pass
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No inherits allowed in config files")
+ return -1
-cdef public void e_inherit(lex_t* c, char* file):
- print "e_inherit", file
- pass
+ return 0
cdef public void e_include(lex_t* c, char* file):
- print "e_include", file
+ from bb.parse import handle
d = <object>c.data
- d.expand(file)
-
+
try:
- parsefile(file, d)
+ handle(d.expand(file,d), d, True)
except IOError:
- print "Could not include required file %s" % file
+ print "Could not include file", file
-cdef public void e_require(lex_t* c, char* file):
- print "e_require", file
+cdef public int e_require(lex_t* c, char* file) except -1:
+ #print "e_require", file
+ from bb.parse import handle
d = <object>c.data
- d.expand(file)
-
+
try:
- parsefile(file, d)
+ handle(d.expand(file,d), d, True)
except IOError:
- raise CParseError("Could not include required file %s" % file)
+ print "ParseError", file
+ from bb.parse import ParseError
+ raise ParseError("Could not include required file %s" % file)
+ return -1
+
+ return 0
+
+cdef public int e_proc(lex_t* c, char* key, char* what) except -1:
+ #print "e_proc", key, what
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No inherits allowed in config files")
+ return -1
+
+ return 0
+
+cdef public int e_proc_python(lex_t* c, char* key, char* what) except -1:
+ #print "e_proc_python"
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No pythin allowed in config files")
+ return -1
+
+ if key != NULL:
+ pass
+ #print "Key", key
+ if what != NULL:
+ pass
+ #print "What", what
+
+ return 0
+
+cdef public int e_proc_fakeroot(lex_t* c, char* key, char* what) except -1:
+ #print "e_fakeroot", key, what
+
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No fakeroot allowed in config files")
+ return -1
+
+ return 0
+
+cdef public int e_def(lex_t* c, char* a, char* b, char* d) except -1:
+ #print "e_def", a, b, d
-cdef public void e_proc(lex_t* c, char* key, char* what):
- print "e_proc", key, what
- pass
+ if c.config == 1:
+ from bb.parse import ParseError
+ raise ParseError("No defs allowed in config files")
+ return -1
-cdef public void e_proc_python(lex_t* c, char* key, char* what):
- print "e_proc_python", key, what
- pass
+ return 0
-cdef public void e_proc_fakeroot(lex_t* c, char* key, char* what):
- print "e_fakeroot", key, what
- pass
+cdef public int e_parse_error(lex_t* c) except -1:
+ print "e_parse_error", c.name, "line:", lineError, "parse:", errorParse
-cdef public void e_def(lex_t* c, char* a, char* b, char* d):
- print "e_def", key, what
- pass
-cdef public void e_parse_error(lex_t* c):
- print "e_parse_error", "line:", lineError, "parse:", errorParse
- raise CParseError("There was an parse error, sorry unable to give more information at the current time.")
+ from bb.parse import ParseError
+ raise ParseError("There was an parse error, sorry unable to give more information at the current time. File: %s Line: %d" % (c.name,lineError) )
+ return -1
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.cc b/bitbake/lib/bb/parse/parse_c/bitbakeparser.cc
index ee9a901b7..9d9793f8d 100644
--- a/bitbake/lib/bb/parse/parse_c/bitbakeparser.cc
+++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.cc
@@ -128,51 +128,49 @@ typedef union {
*/
static const YYACTIONTYPE yy_action[] = {
/* 0 */ 82, 3, 7, 8, 38, 22, 39, 24, 26, 32,
- /* 10 */ 34, 28, 30, 128, 1, 40, 53, 70, 55, 5,
- /* 20 */ 60, 65, 67, 2, 21, 36, 69, 77, 9, 7,
- /* 30 */ 11, 6, 13, 15, 17, 19, 12, 52, 50, 4,
- /* 40 */ 74, 42, 46, 59, 57, 10, 64, 62, 38, 14,
- /* 50 */ 73, 16, 38, 38, 76, 81, 18, 20, 23, 25,
- /* 60 */ 27, 29, 31, 33, 35, 37, 56, 51, 90, 54,
- /* 70 */ 58, 71, 41, 43, 63, 45, 44, 47, 72, 48,
- /* 80 */ 75, 78, 80, 61, 90, 49, 66, 90, 90, 68,
- /* 90 */ 90, 90, 90, 90, 90, 79,
+ /* 10 */ 34, 28, 30, 2, 21, 40, 53, 70, 55, 44,
+ /* 20 */ 60, 65, 67, 128, 1, 36, 69, 77, 42, 46,
+ /* 30 */ 11, 66, 13, 15, 17, 19, 64, 62, 9, 7,
+ /* 40 */ 74, 38, 45, 81, 59, 57, 38, 38, 73, 76,
+ /* 50 */ 5, 68, 52, 50, 14, 31, 47, 71, 48, 10,
+ /* 60 */ 72, 33, 23, 49, 6, 41, 51, 78, 75, 16,
+ /* 70 */ 4, 54, 35, 25, 18, 80, 79, 56, 27, 37,
+ /* 80 */ 58, 12, 61, 29, 43, 63, 20,
};
static const YYCODETYPE yy_lookahead[] = {
/* 0 */ 0, 1, 2, 3, 23, 4, 25, 6, 7, 8,
- /* 10 */ 9, 10, 11, 31, 32, 15, 16, 1, 18, 42,
- /* 20 */ 20, 21, 22, 33, 34, 24, 26, 27, 1, 2,
- /* 30 */ 4, 28, 6, 7, 8, 9, 5, 35, 36, 29,
- /* 40 */ 24, 13, 14, 37, 38, 34, 39, 40, 23, 5,
- /* 50 */ 25, 5, 23, 23, 25, 25, 5, 5, 5, 5,
- /* 60 */ 5, 5, 5, 5, 5, 41, 17, 35, 43, 1,
- /* 70 */ 37, 24, 12, 12, 39, 12, 14, 12, 41, 13,
- /* 80 */ 41, 1, 41, 19, 43, 12, 19, 43, 43, 19,
- /* 90 */ 43, 43, 43, 43, 43, 24,
+ /* 10 */ 9, 10, 11, 33, 34, 15, 16, 1, 18, 14,
+ /* 20 */ 20, 21, 22, 31, 32, 24, 26, 27, 13, 14,
+ /* 30 */ 4, 19, 6, 7, 8, 9, 39, 40, 1, 2,
+ /* 40 */ 24, 23, 12, 25, 37, 38, 23, 23, 25, 25,
+ /* 50 */ 42, 19, 35, 36, 5, 5, 12, 24, 13, 34,
+ /* 60 */ 41, 5, 5, 12, 28, 12, 35, 1, 41, 5,
+ /* 70 */ 29, 1, 5, 5, 5, 41, 24, 17, 5, 41,
+ /* 80 */ 37, 5, 19, 5, 12, 39, 5,
};
#define YY_SHIFT_USE_DFLT (-20)
static const signed char yy_shift_ofst[] = {
- /* 0 */ -20, 0, -20, 10, -20, 3, -20, -20, 27, -20,
- /* 10 */ 26, 31, -20, 44, -20, 46, -20, 51, -20, 52,
- /* 20 */ -20, 1, 53, -20, 54, -20, 55, -20, 56, -20,
- /* 30 */ 57, -20, 58, -20, 59, -20, -20, -19, -20, -20,
- /* 40 */ 60, 28, 61, 62, 63, -20, 65, 66, 73, -20,
- /* 50 */ 60, -20, -20, 68, -20, 49, -20, 49, -20, -20,
- /* 60 */ 64, -20, 64, -20, -20, 67, -20, 70, -20, 16,
- /* 70 */ 47, -20, 25, -20, -20, 29, -20, 80, 71, -20,
- /* 80 */ 30, -20,
+ /* 0 */ -20, 0, -20, 41, -20, 36, -20, -20, 37, -20,
+ /* 10 */ 26, 76, -20, 49, -20, 64, -20, 69, -20, 81,
+ /* 20 */ -20, 1, 57, -20, 68, -20, 73, -20, 78, -20,
+ /* 30 */ 50, -20, 56, -20, 67, -20, -20, -19, -20, -20,
+ /* 40 */ 53, 15, 72, 5, 30, -20, 44, 45, 51, -20,
+ /* 50 */ 53, -20, -20, 70, -20, 60, -20, 60, -20, -20,
+ /* 60 */ 63, -20, 63, -20, -20, 12, -20, 32, -20, 16,
+ /* 70 */ 33, -20, 23, -20, -20, 24, -20, 66, 52, -20,
+ /* 80 */ 18, -20,
};
-#define YY_REDUCE_USE_DFLT (-24)
+#define YY_REDUCE_USE_DFLT (-21)
static const signed char yy_reduce_ofst[] = {
- /* 0 */ -18, -10, -24, -24, -23, -24, -24, -24, 11, -24,
- /* 10 */ -24, -24, -24, -24, -24, -24, -24, -24, -24, -24,
- /* 20 */ -24, -24, -24, -24, -24, -24, -24, -24, -24, -24,
- /* 30 */ -24, -24, -24, -24, -24, -24, 24, -24, -24, -24,
- /* 40 */ 2, -24, -24, -24, -24, -24, -24, -24, -24, -24,
- /* 50 */ 32, -24, -24, -24, -24, 6, -24, 33, -24, -24,
- /* 60 */ 7, -24, 35, -24, -24, -24, -24, -24, -24, -24,
- /* 70 */ -24, 37, -24, -24, 39, -24, -24, -24, -24, 41,
- /* 80 */ -24, -24,
+ /* 0 */ -8, -20, -21, -21, 8, -21, -21, -21, 25, -21,
+ /* 10 */ -21, -21, -21, -21, -21, -21, -21, -21, -21, -21,
+ /* 20 */ -21, -21, -21, -21, -21, -21, -21, -21, -21, -21,
+ /* 30 */ -21, -21, -21, -21, -21, -21, 38, -21, -21, -21,
+ /* 40 */ 17, -21, -21, -21, -21, -21, -21, -21, -21, -21,
+ /* 50 */ 31, -21, -21, -21, -21, 7, -21, 43, -21, -21,
+ /* 60 */ -3, -21, 46, -21, -21, -21, -21, -21, -21, -21,
+ /* 70 */ -21, 19, -21, -21, 27, -21, -21, -21, -21, 34,
+ /* 80 */ -21, -21,
};
static const YYACTIONTYPE yy_default[] = {
/* 0 */ 84, 127, 83, 85, 125, 126, 124, 86, 127, 85,
@@ -420,7 +418,7 @@ static void yy_destructor(YYCODETYPE yymajor, YYMINORTYPE *yypminor){
case 29:
#line 50 "bitbakeparser.y"
{ (yypminor->yy0).release_this (); }
-#line 425 "bitbakeparser.c"
+#line 423 "bitbakeparser.c"
break;
default: break; /* If no destructor action specified: do nothing */
}
@@ -694,7 +692,7 @@ static void yy_reduce(
{ yygotominor.yy0.assignString( (char*)yymsp[0].minor.yy0.string() );
yymsp[0].minor.yy0.assignString( 0 );
yymsp[0].minor.yy0.release_this(); }
-#line 699 "bitbakeparser.c"
+#line 697 "bitbakeparser.c"
break;
case 4:
#line 64 "bitbakeparser.y"
@@ -702,7 +700,7 @@ static void yy_reduce(
yygotominor.yy0.assignString( (char*)yymsp[0].minor.yy0.string() );
yymsp[0].minor.yy0.assignString( 0 );
yymsp[0].minor.yy0.release_this(); }
-#line 707 "bitbakeparser.c"
+#line 705 "bitbakeparser.c"
break;
case 5:
#line 70 "bitbakeparser.y"
@@ -711,7 +709,7 @@ static void yy_reduce(
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(3,&yymsp[-3].minor);
yy_destructor(4,&yymsp[-1].minor);
}
-#line 716 "bitbakeparser.c"
+#line 714 "bitbakeparser.c"
break;
case 6:
#line 74 "bitbakeparser.y"
@@ -720,7 +718,7 @@ static void yy_reduce(
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(3,&yymsp[-3].minor);
yy_destructor(6,&yymsp[-1].minor);
}
-#line 725 "bitbakeparser.c"
+#line 723 "bitbakeparser.c"
break;
case 7:
#line 78 "bitbakeparser.y"
@@ -729,7 +727,7 @@ static void yy_reduce(
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(3,&yymsp[-3].minor);
yy_destructor(7,&yymsp[-1].minor);
}
-#line 734 "bitbakeparser.c"
+#line 732 "bitbakeparser.c"
break;
case 8:
#line 82 "bitbakeparser.y"
@@ -738,7 +736,7 @@ static void yy_reduce(
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(3,&yymsp[-3].minor);
yy_destructor(8,&yymsp[-1].minor);
}
-#line 743 "bitbakeparser.c"
+#line 741 "bitbakeparser.c"
break;
case 9:
#line 86 "bitbakeparser.y"
@@ -746,56 +744,56 @@ static void yy_reduce(
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(3,&yymsp[-3].minor);
yy_destructor(9,&yymsp[-1].minor);
}
-#line 751 "bitbakeparser.c"
+#line 749 "bitbakeparser.c"
break;
case 10:
#line 90 "bitbakeparser.y"
{ e_assign( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(4,&yymsp[-1].minor);
}
-#line 758 "bitbakeparser.c"
+#line 756 "bitbakeparser.c"
break;
case 11:
#line 93 "bitbakeparser.y"
{ e_precat( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(6,&yymsp[-1].minor);
}
-#line 765 "bitbakeparser.c"
+#line 763 "bitbakeparser.c"
break;
case 12:
#line 96 "bitbakeparser.y"
{ e_postcat( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(7,&yymsp[-1].minor);
}
-#line 772 "bitbakeparser.c"
+#line 770 "bitbakeparser.c"
break;
case 13:
#line 99 "bitbakeparser.y"
{ e_prepend( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(10,&yymsp[-1].minor);
}
-#line 779 "bitbakeparser.c"
+#line 777 "bitbakeparser.c"
break;
case 14:
#line 102 "bitbakeparser.y"
{ e_append( lex, yymsp[-2].minor.yy0.string() , yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(11,&yymsp[-1].minor);
}
-#line 786 "bitbakeparser.c"
+#line 784 "bitbakeparser.c"
break;
case 15:
#line 105 "bitbakeparser.y"
{ e_immediate( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(8,&yymsp[-1].minor);
}
-#line 793 "bitbakeparser.c"
+#line 791 "bitbakeparser.c"
break;
case 16:
#line 108 "bitbakeparser.y"
{ e_cond( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string() );
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(9,&yymsp[-1].minor);
}
-#line 800 "bitbakeparser.c"
+#line 798 "bitbakeparser.c"
break;
case 17:
#line 112 "bitbakeparser.y"
@@ -803,7 +801,7 @@ static void yy_reduce(
yymsp[-4].minor.yy0.release_this(); yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(13,&yymsp[-3].minor);
yy_destructor(14,&yymsp[-1].minor);
}
-#line 808 "bitbakeparser.c"
+#line 806 "bitbakeparser.c"
break;
case 18:
#line 115 "bitbakeparser.y"
@@ -811,55 +809,55 @@ static void yy_reduce(
yymsp[-4].minor.yy0.release_this(); yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(14,&yymsp[-3].minor);
yy_destructor(13,&yymsp[-1].minor);
}
-#line 816 "bitbakeparser.c"
+#line 814 "bitbakeparser.c"
break;
case 19:
#line 118 "bitbakeparser.y"
{ e_addtask( lex, yymsp[0].minor.yy0.string(), NULL, NULL);
yymsp[0].minor.yy0.release_this();}
-#line 822 "bitbakeparser.c"
+#line 820 "bitbakeparser.c"
break;
case 20:
#line 121 "bitbakeparser.y"
{ e_addtask( lex, yymsp[-2].minor.yy0.string(), yymsp[0].minor.yy0.string(), NULL);
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(13,&yymsp[-1].minor);
}
-#line 829 "bitbakeparser.c"
+#line 827 "bitbakeparser.c"
break;
case 21:
#line 124 "bitbakeparser.y"
{ e_addtask( lex, yymsp[-2].minor.yy0.string(), NULL, yymsp[0].minor.yy0.string());
yymsp[-2].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); yy_destructor(14,&yymsp[-1].minor);
}
-#line 836 "bitbakeparser.c"
+#line 834 "bitbakeparser.c"
break;
case 25:
#line 131 "bitbakeparser.y"
{ e_addhandler( lex, yymsp[0].minor.yy0.string()); yymsp[0].minor.yy0.release_this (); yy_destructor(16,&yymsp[-1].minor);
}
-#line 842 "bitbakeparser.c"
+#line 840 "bitbakeparser.c"
break;
case 26:
#line 133 "bitbakeparser.y"
{ e_export_func( lex, yymsp[0].minor.yy0.string()); yymsp[0].minor.yy0.release_this(); }
-#line 847 "bitbakeparser.c"
+#line 845 "bitbakeparser.c"
break;
case 30:
#line 138 "bitbakeparser.y"
{ e_inherit( lex, yymsp[0].minor.yy0.string() ); yymsp[0].minor.yy0.release_this (); }
-#line 852 "bitbakeparser.c"
+#line 850 "bitbakeparser.c"
break;
case 34:
#line 144 "bitbakeparser.y"
{ e_include( lex, yymsp[0].minor.yy0.string() ); yymsp[0].minor.yy0.release_this(); yy_destructor(21,&yymsp[-1].minor);
}
-#line 858 "bitbakeparser.c"
+#line 856 "bitbakeparser.c"
break;
case 35:
#line 147 "bitbakeparser.y"
{ e_require( lex, yymsp[0].minor.yy0.string() ); yymsp[0].minor.yy0.release_this(); yy_destructor(22,&yymsp[-1].minor);
}
-#line 864 "bitbakeparser.c"
+#line 862 "bitbakeparser.c"
break;
case 36:
#line 150 "bitbakeparser.y"
@@ -868,12 +866,12 @@ static void yy_reduce(
yymsp[-1].minor.yy0.release_this ();
yymsp[0].minor.yy0.release_this ();
}
-#line 873 "bitbakeparser.c"
+#line 871 "bitbakeparser.c"
break;
case 37:
#line 155 "bitbakeparser.y"
{ yygotominor.yy0.assignString(0); }
-#line 878 "bitbakeparser.c"
+#line 876 "bitbakeparser.c"
break;
case 38:
#line 157 "bitbakeparser.y"
@@ -881,7 +879,7 @@ static void yy_reduce(
yymsp[-3].minor.yy0.release_this(); yymsp[-1].minor.yy0.release_this(); yy_destructor(24,&yymsp[-2].minor);
yy_destructor(25,&yymsp[0].minor);
}
-#line 886 "bitbakeparser.c"
+#line 884 "bitbakeparser.c"
break;
case 39:
#line 160 "bitbakeparser.y"
@@ -890,7 +888,7 @@ static void yy_reduce(
yy_destructor(24,&yymsp[-2].minor);
yy_destructor(25,&yymsp[0].minor);
}
-#line 895 "bitbakeparser.c"
+#line 893 "bitbakeparser.c"
break;
case 40:
#line 163 "bitbakeparser.y"
@@ -899,7 +897,7 @@ static void yy_reduce(
yy_destructor(24,&yymsp[-2].minor);
yy_destructor(25,&yymsp[0].minor);
}
-#line 904 "bitbakeparser.c"
+#line 902 "bitbakeparser.c"
break;
case 41:
#line 167 "bitbakeparser.y"
@@ -908,7 +906,7 @@ static void yy_reduce(
yy_destructor(24,&yymsp[-2].minor);
yy_destructor(25,&yymsp[0].minor);
}
-#line 913 "bitbakeparser.c"
+#line 911 "bitbakeparser.c"
break;
case 42:
#line 171 "bitbakeparser.y"
@@ -916,18 +914,18 @@ static void yy_reduce(
yygotominor.yy0.assignString( token_t::concatString(yymsp[-1].minor.yy0.string(), yymsp[0].minor.yy0.string()) );
yymsp[-1].minor.yy0.release_this (); yymsp[0].minor.yy0.release_this ();
}
-#line 921 "bitbakeparser.c"
+#line 919 "bitbakeparser.c"
break;
case 43:
#line 175 "bitbakeparser.y"
{ yygotominor.yy0.assignString( 0 ); }
-#line 926 "bitbakeparser.c"
+#line 924 "bitbakeparser.c"
break;
case 44:
#line 177 "bitbakeparser.y"
{ e_def( lex, yymsp[-2].minor.yy0.string(), yymsp[-1].minor.yy0.string(), yymsp[0].minor.yy0.string());
yymsp[-2].minor.yy0.release_this(); yymsp[-1].minor.yy0.release_this(); yymsp[0].minor.yy0.release_this(); }
-#line 932 "bitbakeparser.c"
+#line 930 "bitbakeparser.c"
break;
};
yygoto = yyRuleInfo[yyruleno].lhs;
@@ -986,7 +984,7 @@ static void yy_syntax_error(
#define TOKEN (yyminor.yy0)
#line 52 "bitbakeparser.y"
e_parse_error( lex );
-#line 992 "bitbakeparser.c"
+#line 990 "bitbakeparser.c"
bbparseARG_STORE; /* Suppress warning about unused %extra_argument variable */
}
@@ -1042,7 +1040,7 @@ void bbparse(
/* (re)initialize the parser, if necessary */
yypParser = (yyParser*)yyp;
if( yypParser->yyidx<0 ){
- /* if( yymajor==0 ) return; // not sure why this was here... */
+ if( yymajor==0 ) return;
yypParser->yyidx = 0;
yypParser->yyerrcnt = -1;
yypParser->yystack[0].stateno = 0;
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakescanner.cc b/bitbake/lib/bb/parse/parse_c/bitbakescanner.cc
index 43dad12d3..acc13f7c3 100644
--- a/bitbake/lib/bb/parse/parse_c/bitbakescanner.cc
+++ b/bitbake/lib/bb/parse/parse_c/bitbakescanner.cc
@@ -8,7 +8,7 @@
#define FLEX_SCANNER
#define YY_FLEX_MAJOR_VERSION 2
#define YY_FLEX_MINOR_VERSION 5
-#define YY_FLEX_SUBMINOR_VERSION 31
+#define YY_FLEX_SUBMINOR_VERSION 33
#if YY_FLEX_SUBMINOR_VERSION > 0
#define FLEX_BETA
#endif
@@ -30,7 +30,15 @@
/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
-#if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L
+#if __STDC_VERSION__ >= 199901L
+
+/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
+ * if you want the limit (max/min) macros for int types.
+ */
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS 1
+#endif
+
#include <inttypes.h>
typedef int8_t flex_int8_t;
typedef uint8_t flex_uint8_t;
@@ -153,6 +161,10 @@ int yylex_init (yyscan_t* scanner);
#define YY_BUF_SIZE 16384
#endif
+/* The state buf must be large enough to hold one state per character in the main buffer.
+ */
+#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
+
#ifndef YY_TYPEDEF_YY_BUFFER_STATE
#define YY_TYPEDEF_YY_BUFFER_STATE
typedef struct yy_buffer_state *YY_BUFFER_STATE;
@@ -493,7 +505,7 @@ static yyconst flex_int32_t yy_ec[256] =
static yyconst flex_int32_t yy_meta[59] =
{ 0,
1, 1, 2, 3, 1, 1, 4, 1, 1, 1,
- 5, 6, 5, 5, 7, 8, 1, 7, 1, 9,
+ 5, 6, 5, 5, 5, 7, 1, 8, 1, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 10, 1, 11, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
@@ -565,18 +577,18 @@ static yyconst flex_int16_t yy_base[847] =
2077, 2072, 2066, 2069, 2056, 2067, 1398, 1343, 1408, 1404,
643, 1409, 2071, 2066, 2060, 2063, 2050, 2061, 2065, 2060,
2054, 2057, 2044, 2055, 1420, 1445, 1413, 1447, 1453, 1454,
- 2059, 2054, 2047, 2050, 2035, 2043, 1455, 1459, 1460, 1461,
+ 2059, 2053, 2047, 2049, 2032, 2043, 1455, 1459, 1460, 1461,
1462, 1463, 1471, 1436, 1430, 1192, 1433, 1479, 1482, 1492,
- 1506, 1519, 1520, 1528, 2047, 2040, 2031, 0, 2034, 2019,
- 2027, 1486, 1496, 1505, 1506, 1510, 1516, 1524, 2044, 2018,
- 0, 0, 0, 0, 1281, 1517, 2043, 2042, 2039, 2035,
- 2023, 1994, 2309, 2309, 2309, 2309, 2005, 1981, 0, 0,
+ 1506, 1519, 1520, 1528, 2046, 2037, 2031, 0, 2033, 2016,
+ 2027, 1486, 1496, 1505, 1506, 1510, 1516, 1524, 2043, 2015,
+ 0, 0, 0, 0, 1281, 1517, 2043, 2041, 2036, 2034,
+ 2024, 1995, 2309, 2309, 2309, 2309, 2005, 1981, 0, 0,
0, 0, 1538, 1528, 1530, 1534, 1537, 1540, 1981, 1957,
0, 0, 0, 0, 1557, 1558, 1559, 1560, 1561, 1563,
- 1568, 1547, 1988, 1959, 1954, 1948, 1580, 1581, 1582, 1590,
- 1592, 1594, 1923, 1863, 0, 0, 0, 0, 1598, 1599,
- 1600, 1874, 1858, 1350, 1584, 1803, 1792, 1801, 1790, 1603,
+ 1568, 1547, 1988, 1959, 1955, 1948, 1580, 1581, 1582, 1590,
+ 1592, 1594, 1924, 1863, 0, 0, 0, 0, 1598, 1599,
+ 1600, 1875, 1859, 1350, 1584, 1803, 1792, 1801, 1790, 1603,
1601, 1799, 1788, 1604, 1602, 1610, 1609, 1643, 1644, 1797,
1786, 1611, 1630, 1800, 1773, 1010, 1606, 1798, 1771, 1795,
@@ -593,8 +605,8 @@ static yyconst flex_int16_t yy_base[847] =
1768, 0, 742, 2309, 0, 1764, 0, 1778, 678, 1801,
0, 2309, 1835, 1847, 1859, 1871, 1883, 550, 1892, 1898,
1907, 1919, 1931, 1939, 1945, 1950, 1956, 1965, 1977, 1989,
- 2001, 2013, 2025, 2033, 2039, 2042, 306, 304, 301, 2049,
- 213, 2057, 136, 2065, 2073, 2081
+ 2001, 2013, 2025, 2033, 2039, 2043, 306, 304, 301, 2050,
+ 213, 2058, 136, 2066, 2074, 2082
} ;
static yyconst flex_int16_t yy_def[847] =
@@ -903,14 +915,14 @@ static yyconst flex_int16_t yy_nxt[2368] =
112, 112, 112, 112, 112, 112, 112, 112, 112, 112,
112, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 167, 167, 167, 705, 167,
+ 155, 155, 155, 155, 155, 167, 167, 167, 167, 705,
- 167, 167, 177, 177, 704, 177, 177, 183, 701, 183,
+ 167, 167, 177, 177, 177, 704, 177, 183, 701, 183,
183, 183, 183, 183, 183, 183, 183, 183, 183, 187,
187, 187, 187, 187, 187, 187, 187, 187, 187, 187,
187, 201, 201, 201, 201, 201, 201, 201, 201, 201,
- 201, 201, 201, 209, 209, 700, 209, 209, 217, 217,
- 238, 217, 217, 217, 223, 223, 238, 223, 223, 231,
+ 201, 201, 201, 209, 209, 209, 700, 209, 217, 217,
+ 238, 217, 217, 217, 223, 223, 223, 238, 223, 231,
231, 238, 231, 231, 231, 237, 237, 237, 237, 237,
237, 237, 237, 237, 237, 237, 237, 239, 239, 239,
239, 239, 239, 239, 239, 239, 239, 239, 239, 256,
@@ -919,13 +931,13 @@ static yyconst flex_int16_t yy_nxt[2368] =
256, 261, 693, 692, 261, 261, 261, 261, 261, 261,
261, 261, 261, 264, 264, 264, 264, 264, 264, 264,
264, 264, 264, 264, 264, 267, 689, 688, 267, 267,
- 267, 267, 267, 267, 267, 267, 267, 284, 284, 687,
- 284, 284, 292, 292, 292, 686, 292, 292, 292, 296,
- 296, 184, 296, 418, 418, 184, 418, 418, 184, 184,
- 418, 433, 433, 683, 433, 433, 682, 678, 433, 465,
- 465, 677, 465, 465, 676, 675, 465, 500, 500, 674,
- 500, 500, 673, 654, 500, 514, 514, 653, 514, 514,
- 652, 651, 514, 650, 649, 642, 641, 640, 639, 638,
+ 267, 267, 267, 267, 267, 267, 267, 284, 284, 284,
+ 687, 284, 292, 292, 292, 292, 686, 292, 292, 296,
+ 184, 296, 184, 296, 418, 418, 418, 184, 418, 184,
+ 683, 418, 433, 433, 433, 682, 433, 678, 677, 433,
+ 465, 465, 465, 676, 465, 675, 674, 465, 500, 500,
+ 500, 673, 500, 654, 653, 500, 514, 514, 514, 652,
+ 514, 651, 650, 514, 649, 642, 641, 640, 639, 638,
637, 636, 635, 634, 633, 632, 631, 624, 623, 622,
621, 620, 619, 611, 610, 609, 608, 607, 606, 605,
@@ -1167,14 +1179,14 @@ static yyconst flex_int16_t yy_chk[2368] =
815, 815, 815, 815, 815, 815, 815, 815, 815, 815,
815, 816, 816, 816, 816, 816, 816, 816, 816, 816,
816, 816, 816, 817, 817, 817, 817, 817, 817, 817,
- 817, 817, 817, 817, 817, 819, 819, 819, 683, 819,
+ 817, 817, 817, 817, 817, 819, 819, 819, 819, 683,
- 819, 819, 820, 820, 682, 820, 820, 821, 674, 821,
+ 819, 819, 820, 820, 820, 682, 820, 821, 674, 821,
821, 821, 821, 821, 821, 821, 821, 821, 821, 822,
822, 822, 822, 822, 822, 822, 822, 822, 822, 822,
822, 823, 823, 823, 823, 823, 823, 823, 823, 823,
- 823, 823, 823, 824, 824, 673, 824, 824, 825, 825,
- 666, 825, 825, 825, 826, 826, 665, 826, 826, 827,
+ 823, 823, 823, 824, 824, 824, 673, 824, 825, 825,
+ 666, 825, 825, 825, 826, 826, 826, 665, 826, 827,
827, 664, 827, 827, 827, 828, 828, 828, 828, 828,
828, 828, 828, 828, 828, 828, 828, 829, 829, 829,
829, 829, 829, 829, 829, 829, 829, 829, 829, 830,
@@ -1183,13 +1195,13 @@ static yyconst flex_int16_t yy_chk[2368] =
830, 831, 650, 649, 831, 831, 831, 831, 831, 831,
831, 831, 831, 832, 832, 832, 832, 832, 832, 832,
832, 832, 832, 832, 832, 833, 638, 637, 833, 833,
- 833, 833, 833, 833, 833, 833, 833, 834, 834, 632,
- 834, 834, 835, 835, 835, 631, 835, 835, 835, 836,
- 836, 630, 836, 840, 840, 629, 840, 840, 628, 627,
- 840, 842, 842, 620, 842, 842, 619, 611, 842, 844,
- 844, 610, 844, 844, 609, 607, 844, 845, 845, 606,
- 845, 845, 605, 586, 845, 846, 846, 585, 846, 846,
- 584, 583, 846, 582, 581, 574, 573, 572, 571, 570,
+ 833, 833, 833, 833, 833, 833, 833, 834, 834, 834,
+ 632, 834, 835, 835, 835, 835, 631, 835, 835, 836,
+ 630, 836, 629, 836, 840, 840, 840, 628, 840, 627,
+ 620, 840, 842, 842, 842, 619, 842, 611, 610, 842,
+ 844, 844, 844, 609, 844, 607, 606, 844, 845, 845,
+ 845, 605, 845, 586, 585, 845, 846, 846, 846, 584,
+ 846, 583, 582, 846, 581, 574, 573, 572, 571, 570,
569, 568, 567, 566, 565, 564, 563, 556, 555, 554,
553, 552, 551, 541, 540, 539, 538, 536, 535, 534,
@@ -1323,7 +1335,7 @@ int errorParse;
enum {
errorNone = 0,
errorUnexpectedInput,
- errorUnsupportedFeature,
+ errorUnsupportedFeature,
};
}
@@ -1351,7 +1363,7 @@ static const char* fixup_escapes (const char* sz);
-#line 1355 "<stdout>"
+#line 1367 "<stdout>"
#define INITIAL 0
#define S_DEF 1
@@ -1587,11 +1599,11 @@ YY_DECL
#line 164 "bitbakescanner.l"
-#line 1591 "<stdout>"
+#line 1603 "<stdout>"
- if ( yyg->yy_init )
+ if ( !yyg->yy_init )
{
- yyg->yy_init = 0;
+ yyg->yy_init = 1;
#ifdef YY_USER_INIT
YY_USER_INIT;
@@ -1972,7 +1984,7 @@ YY_RULE_SETUP
#line 254 "bitbakescanner.l"
ECHO;
YY_BREAK
-#line 1976 "<stdout>"
+#line 1988 "<stdout>"
case YY_END_OF_BUFFER:
{
@@ -2274,7 +2286,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner)
{
register int yy_is_jam;
- struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+ struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */
register char *yy_cp = yyg->yy_c_buf_p;
register YY_CHAR yy_c = 1;
@@ -2730,10 +2742,10 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann
* @note If you want to scan bytes that may contain NUL values, then use
* yy_scan_bytes() instead.
*/
-YY_BUFFER_STATE yy_scan_string (yyconst char * yy_str , yyscan_t yyscanner)
+YY_BUFFER_STATE yy_scan_string (yyconst char * yystr , yyscan_t yyscanner)
{
- return yy_scan_bytes(yy_str,strlen(yy_str) ,yyscanner);
+ return yy_scan_bytes(yystr,strlen(yystr) ,yyscanner);
}
/** Setup the input buffer state to scan the given bytes. The next call to yylex() will
@@ -2743,7 +2755,7 @@ YY_BUFFER_STATE yy_scan_string (yyconst char * yy_str , yyscan_t yyscanner)
* @param yyscanner The scanner object.
* @return the newly allocated buffer state object.
*/
-YY_BUFFER_STATE yy_scan_bytes (yyconst char * bytes, int len , yyscan_t yyscanner)
+YY_BUFFER_STATE yy_scan_bytes (yyconst char * yybytes, int _yybytes_len , yyscan_t yyscanner)
{
YY_BUFFER_STATE b;
char *buf;
@@ -2751,15 +2763,15 @@ YY_BUFFER_STATE yy_scan_bytes (yyconst char * bytes, int len , yyscan_t yyscan
int i;
/* Get memory for full buffer, including space for trailing EOB's. */
- n = len + 2;
+ n = _yybytes_len + 2;
buf = (char *) yyalloc(n ,yyscanner );
if ( ! buf )
YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" );
- for ( i = 0; i < len; ++i )
- buf[i] = bytes[i];
+ for ( i = 0; i < _yybytes_len; ++i )
+ buf[i] = yybytes[i];
- buf[len] = buf[len+1] = YY_END_OF_BUFFER_CHAR;
+ buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR;
b = yy_scan_buffer(buf,n ,yyscanner);
if ( ! b )
@@ -2987,21 +2999,51 @@ void yyset_debug (int bdebug , yyscan_t yyscanner)
/* Accessor methods for yylval and yylloc */
+/* User-visible API */
+
+/* yylex_init is special because it creates the scanner itself, so it is
+ * the ONLY reentrant function that doesn't take the scanner as the last argument.
+ * That's why we explicitly handle the declaration, instead of using our macros.
+ */
+
+int yylex_init(yyscan_t* ptr_yy_globals)
+
+{
+ if (ptr_yy_globals == NULL){
+ errno = EINVAL;
+ return 1;
+ }
+
+ *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), NULL );
+
+ if (*ptr_yy_globals == NULL){
+ errno = ENOMEM;
+ return 1;
+ }
+
+ /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */
+ memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t));
+
+ return yy_init_globals ( *ptr_yy_globals );
+}
+
static int yy_init_globals (yyscan_t yyscanner)
{
struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
/* Initialization is the same as for the non-reentrant scanner.
- This function is called once per scanner lifetime. */
+ * This function is called from yylex_destroy(), so don't allocate here.
+ */
yyg->yy_buffer_stack = 0;
yyg->yy_buffer_stack_top = 0;
yyg->yy_buffer_stack_max = 0;
yyg->yy_c_buf_p = (char *) 0;
- yyg->yy_init = 1;
+ yyg->yy_init = 0;
yyg->yy_start = 0;
+
yyg->yy_start_stack_ptr = 0;
yyg->yy_start_stack_depth = 0;
- yyg->yy_start_stack = (int *) 0;
+ yyg->yy_start_stack = NULL;
/* Defined in main.c */
#ifdef YY_STDINIT
@@ -3018,33 +3060,6 @@ static int yy_init_globals (yyscan_t yyscanner)
return 0;
}
-/* User-visible API */
-
-/* yylex_init is special because it creates the scanner itself, so it is
- * the ONLY reentrant function that doesn't take the scanner as the last argument.
- * That's why we explicitly handle the declaration, instead of using our macros.
- */
-
-int yylex_init(yyscan_t* ptr_yy_globals)
-
-{
- if (ptr_yy_globals == NULL){
- errno = EINVAL;
- return 1;
- }
-
- *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), NULL );
-
- if (*ptr_yy_globals == NULL){
- errno = ENOMEM;
- return 1;
- }
-
- memset(*ptr_yy_globals,0,sizeof(struct yyguts_t));
-
- return yy_init_globals ( *ptr_yy_globals );
-}
-
/* yylex_destroy is for both reentrant and non-reentrant scanners. */
int yylex_destroy (yyscan_t yyscanner)
{
@@ -3065,8 +3080,13 @@ int yylex_destroy (yyscan_t yyscanner)
yyfree(yyg->yy_start_stack ,yyscanner );
yyg->yy_start_stack = NULL;
+ /* Reset the globals. This is important in a non-reentrant scanner so the next time
+ * yylex() is called, initialization will occur. */
+ yy_init_globals( yyscanner);
+
/* Destroy the main struct (reentrant only). */
yyfree ( yyscanner , yyscanner );
+ yyscanner = NULL;
return 0;
}
@@ -3078,7 +3098,6 @@ int yylex_destroy (yyscan_t yyscanner)
static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner)
{
register int i;
- struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
for ( i = 0; i < n; ++i )
s1[i] = s2[i];
}
@@ -3088,7 +3107,6 @@ static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yysca
static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner)
{
register int n;
- struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
for ( n = 0; s[n]; ++n )
;
@@ -3120,18 +3138,6 @@ void yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
-#undef YY_NEW_FILE
-#undef YY_FLUSH_BUFFER
-#undef yy_set_bol
-#undef yy_new_buffer
-#undef yy_set_interactive
-#undef yytext_ptr
-#undef YY_DO_BEFORE_ACTION
-
-#ifdef YY_DECL_IS_OURS
-#undef YY_DECL_IS_OURS
-#undef YY_DECL
-#endif
#line 254 "bitbakescanner.l"
@@ -3148,47 +3154,49 @@ void lex_t::accept (int token, const char* sz)
void lex_t::input (char *buf, int *result, int max_size)
{
- printf("lex_t::input %p %d\n", buf, max_size);
+ /* printf("lex_t::input %p %d\n", buf, max_size); */
*result = fread(buf, 1, max_size, file);
- printf("lex_t::input result %d\n", *result);
+ /* printf("lex_t::input result %d\n", *result); */
}
int lex_t::line ()const
{
- printf("lex_t::line\n");
+ /* printf("lex_t::line\n"); */
return yyget_lineno (scanner);
}
extern "C" {
- void parse (FILE* file, PyObject* data)
+ void parse (FILE* file, char* name, PyObject* data, int config)
{
- printf("parse bbparseAlloc\n");
+ /* printf("parse bbparseAlloc\n"); */
void* parser = bbparseAlloc (malloc);
yyscan_t scanner;
lex_t lex;
- printf("parse yylex_init\n");
+ /* printf("parse yylex_init\n"); */
yylex_init (&scanner);
lex.parser = parser;
lex.scanner = scanner;
lex.file = file;
+ lex.name = name;
lex.data = data;
+ lex.config = config;
lex.parse = bbparse;
- printf("parse yyset_extra\n");
+ /*printf("parse yyset_extra\n"); */
yyset_extra (&lex, scanner);
- printf("parse yylex\n");
+ /* printf("parse yylex\n"); */
int result = yylex (scanner);
-
- printf("parse result %d\n", result);
+
+ /* printf("parse result %d\n", result); */
lex.accept (0);
- printf("parse lex.accept\n");
+ /* printf("parse lex.accept\n"); */
bbparseTrace (NULL, NULL);
- printf("parse bbparseTrace\n");
+ /* printf("parse bbparseTrace\n"); */
if (result != T_EOF)
printf ("premature end of file\n");
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakescanner.l b/bitbake/lib/bb/parse/parse_c/bitbakescanner.l
index f69a7325c..b6592f28e 100644
--- a/bitbake/lib/bb/parse/parse_c/bitbakescanner.l
+++ b/bitbake/lib/bb/parse/parse_c/bitbakescanner.l
@@ -91,7 +91,7 @@ int errorParse;
enum {
errorNone = 0,
errorUnexpectedInput,
- errorUnsupportedFeature,
+ errorUnsupportedFeature,
};
}
@@ -142,7 +142,7 @@ SSTRING \'([^\n\r]|"\\\n")*\'
VALUE ([^'" \t\n])|([^'" \t\n]([^\n]|(\\\n))*[^'" \t\n])
C_SS [a-zA-Z_]
-C_SB [a-zA-Z0-9_+-.]
+C_SB [a-zA-Z0-9_+-./]
REF $\{{C_SS}{C_SB}*\}
SYMBOL {C_SS}{C_SB}*
VARIABLE $?{C_SS}({C_SB}*|{REF})*(\[[a-zA-Z0-9_]*\])?
@@ -265,47 +265,49 @@ void lex_t::accept (int token, const char* sz)
void lex_t::input (char *buf, int *result, int max_size)
{
- printf("lex_t::input %p %d\n", buf, max_size);
+ /* printf("lex_t::input %p %d\n", buf, max_size); */
*result = fread(buf, 1, max_size, file);
- printf("lex_t::input result %d\n", *result);
+ /* printf("lex_t::input result %d\n", *result); */
}
int lex_t::line ()const
{
- printf("lex_t::line\n");
+ /* printf("lex_t::line\n"); */
return yyget_lineno (scanner);
}
extern "C" {
- void parse (FILE* file, PyObject* data)
+ void parse (FILE* file, char* name, PyObject* data, int config)
{
- printf("parse bbparseAlloc\n");
+ /* printf("parse bbparseAlloc\n"); */
void* parser = bbparseAlloc (malloc);
yyscan_t scanner;
lex_t lex;
- printf("parse yylex_init\n");
+ /* printf("parse yylex_init\n"); */
yylex_init (&scanner);
lex.parser = parser;
lex.scanner = scanner;
lex.file = file;
+ lex.name = name;
lex.data = data;
+ lex.config = config;
lex.parse = bbparse;
- printf("parse yyset_extra\n");
+ /*printf("parse yyset_extra\n"); */
yyset_extra (&lex, scanner);
- printf("parse yylex\n");
+ /* printf("parse yylex\n"); */
int result = yylex (scanner);
-
- printf("parse result %d\n", result);
+
+ /* printf("parse result %d\n", result); */
lex.accept (0);
- printf("parse lex.accept\n");
+ /* printf("parse lex.accept\n"); */
bbparseTrace (NULL, NULL);
- printf("parse bbparseTrace\n");
+ /* printf("parse bbparseTrace\n"); */
if (result != T_EOF)
printf ("premature end of file\n");
diff --git a/bitbake/lib/bb/parse/parse_c/lexer.h b/bitbake/lib/bb/parse/parse_c/lexer.h
index 651f3a861..cb32be703 100644
--- a/bitbake/lib/bb/parse/parse_c/lexer.h
+++ b/bitbake/lib/bb/parse/parse_c/lexer.h
@@ -27,13 +27,15 @@ THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#include "Python.h"
extern "C" {
-
+
struct lex_t {
void* parser;
void* scanner;
- FILE* file;
+ FILE* file;
+ char *name;
PyObject *data;
-
+ int config;
+
void* (*parse)(void*, int, token_t, lex_t*);
void accept(int token, const char* sz = NULL);
diff --git a/bitbake/lib/bb/parse/parse_c/lexerc.h b/bitbake/lib/bb/parse/parse_c/lexerc.h
index 0163a7d63..c8a19fb22 100644
--- a/bitbake/lib/bb/parse/parse_c/lexerc.h
+++ b/bitbake/lib/bb/parse/parse_c/lexerc.h
@@ -11,7 +11,9 @@ typedef struct {
void *parser;
void *scanner;
FILE *file;
+ char *name;
PyObject *data;
+ int config;
} lex_t;
#endif
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index c82090fec..34f4d2599 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -23,7 +23,7 @@
import re, bb, os, sys, time
import bb.fetch, bb.build, bb.utils
-from bb import debug, data, fetch, fatal, methodpool
+from bb import data, fetch, methodpool
from ConfHandler import include, localpath, obtain, init
from bb.parse import ParseError
@@ -44,6 +44,13 @@ __bbpath_found__ = 0
__classname__ = ""
classes = [ None, ]
+# We need to indicate EOF to the feeder. This code is so messy that
+# factoring it out to a close_parse_file method is out of question.
+# We will use the IN_PYTHON_EOF as an indicator to just close the method
+#
+# The two parts using it are tightly integrated anyway
+IN_PYTHON_EOF = -9999999999999
+
__parsed_methods__ = methodpool.get_parsed_dict()
def supports(fn, d):
@@ -60,9 +67,9 @@ def inherit(files, d):
file = os.path.join('classes', '%s.bbclass' % file)
if not file in __inherit_cache.split():
- debug(2, "BB %s:%d: inheriting %s" % (fn, lineno, file))
+ bb.msg.debug(2, bb.msg.domain.Parsing, "BB %s:%d: inheriting %s" % (fn, lineno, file))
__inherit_cache += " %s" % file
- include(fn, file, d)
+ include(fn, file, d, "inherit")
data.setVar('__inherit_cache', __inherit_cache, d)
@@ -75,9 +82,9 @@ def handle(fn, d, include = 0):
__residue__ = []
if include == 0:
- debug(2, "BB " + fn + ": handle(data)")
+ bb.msg.debug(2, bb.msg.domain.Parsing, "BB " + fn + ": handle(data)")
else:
- debug(2, "BB " + fn + ": handle(data, include)")
+ bb.msg.debug(2, bb.msg.domain.Parsing, "BB " + fn + ": handle(data, include)")
(root, ext) = os.path.splitext(os.path.basename(fn))
base_name = "%s%s" % (root,ext)
@@ -132,7 +139,7 @@ def handle(fn, d, include = 0):
feeder(lineno, s, fn, base_name, d)
if __inpython__:
# add a blank line to close out any python definition
- feeder(lineno + 1, "", fn, base_name, d)
+ feeder(IN_PYTHON_EOF, "", fn, base_name, d)
if ext == ".bbclass":
classes.remove(__classname__)
else:
@@ -152,7 +159,7 @@ def handle(fn, d, include = 0):
if t:
data.setVar('T', t, d)
except Exception, e:
- bb.debug(1, "executing anonymous function: %s" % e)
+ bb.msg.debug(1, bb.msg.domain.Parsing, "executing anonymous function: %s" % e)
raise
data.delVar("__anonqueue", d)
data.delVar("__anonfunc", d)
@@ -220,7 +227,7 @@ def feeder(lineno, s, fn, root, d):
if __inpython__:
m = __python_func_regexp__.match(s)
- if m:
+ if m and lineno != IN_PYTHON_EOF:
__body__.append(s)
return
else:
@@ -240,6 +247,9 @@ def feeder(lineno, s, fn, root, d):
__body__ = []
__inpython__ = False
+ if lineno == IN_PYTHON_EOF:
+ return
+
# fall through
if s == '' or s[0] == '#': return # skip comments and empty lines
@@ -374,7 +384,7 @@ def vars_from_file(mypkg, d):
def set_additional_vars(file, d, include):
"""Deduce rest of variables, e.g. ${A} out of ${SRC_URI}"""
- debug(2,"BB %s: set_additional_vars" % file)
+ bb.msg.debug(2, bb.msg.domain.Parsing, "BB %s: set_additional_vars" % file)
src_uri = data.getVar('SRC_URI', d)
if not src_uri:
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index 90978300a..4bc2bbc2b 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -22,7 +22,6 @@
Place, Suite 330, Boston, MA 02111-1307 USA."""
import re, bb.data, os, sys
-from bb import debug, fatal
from bb.parse import ParseError
#__config_regexp__ = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}]+)\s*(?P<colon>:)?(?P<ques>\?)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
@@ -53,7 +52,7 @@ def localpath(fn, d):
localfn = fn
return localfn
-def obtain(fn, data = bb.data.init()):
+def obtain(fn, data):
import sys, bb
fn = bb.data.expand(fn, data)
localfn = bb.data.expand(localpath(fn, data), data)
@@ -61,30 +60,30 @@ def obtain(fn, data = bb.data.init()):
if localfn != fn:
dldir = bb.data.getVar('DL_DIR', data, 1)
if not dldir:
- debug(1, "obtain: DL_DIR not defined")
+ bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: DL_DIR not defined")
return localfn
bb.mkdirhier(dldir)
try:
bb.fetch.init([fn])
except bb.fetch.NoMethodError:
(type, value, traceback) = sys.exc_info()
- debug(1, "obtain: no method: %s" % value)
+ bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: no method: %s" % value)
return localfn
try:
bb.fetch.go(data)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
- debug(1, "obtain: missing parameters: %s" % value)
+ bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: missing parameters: %s" % value)
return localfn
except bb.fetch.FetchError:
(type, value, traceback) = sys.exc_info()
- debug(1, "obtain: failed: %s" % value)
+ bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: failed: %s" % value)
return localfn
return localfn
-def include(oldfn, fn, data = bb.data.init(), error_out = False):
+def include(oldfn, fn, data, error_out):
"""
error_out If True a ParseError will be reaised if the to be included
@@ -101,10 +100,10 @@ def include(oldfn, fn, data = bb.data.init(), error_out = False):
ret = handle(fn, data, True)
except IOError:
if error_out:
- raise ParseError("Could not include required file %(fn)s" % vars() )
- debug(2, "CONF file '%s' not found" % fn)
+ raise ParseError("Could not %(error_out)s file %(fn)s" % vars() )
+ bb.msg.debug(2, bb.msg.domain.Parsing, "CONF file '%s' not found" % fn)
-def handle(fn, data = bb.data.init(), include = 0):
+def handle(fn, data, include = 0):
if include:
inc_string = "including"
else:
@@ -129,13 +128,13 @@ def handle(fn, data = bb.data.init(), include = 0):
if os.access(currname, os.R_OK):
f = open(currname, 'r')
abs_fn = currname
- debug(1, "CONF %s %s" % (inc_string, currname))
+ bb.msg.debug(2, bb.msg.domain.Parsing, "CONF %s %s" % (inc_string, currname))
break
if f is None:
raise IOError("file '%s' not found" % fn)
else:
f = open(fn,'r')
- debug(1, "CONF %s %s" % (inc_string,fn))
+ bb.msg.debug(1, bb.msg.domain.Parsing, "CONF %s %s" % (inc_string,fn))
abs_fn = fn
if include:
@@ -161,7 +160,7 @@ def handle(fn, data = bb.data.init(), include = 0):
bb.data.setVar('FILE', oldfile, data)
return data
-def feeder(lineno, s, fn, data = bb.data.init()):
+def feeder(lineno, s, fn, data):
m = __config_regexp__.match(s)
if m:
groupd = m.groupdict()
@@ -185,7 +184,7 @@ def feeder(lineno, s, fn, data = bb.data.init()):
else:
val = groupd["value"]
if 'flag' in groupd and groupd['flag'] != None:
-# bb.note("setVarFlag(%s, %s, %s, data)" % (key, groupd['flag'], val))
+ bb.msg.debug(3, bb.msg.domain.Parsing, "setVarFlag(%s, %s, %s, data)" % (key, groupd['flag'], val))
bb.data.setVarFlag(key, groupd['flag'], val, data)
else:
bb.data.setVar(key, val, data)
@@ -194,14 +193,14 @@ def feeder(lineno, s, fn, data = bb.data.init()):
m = __include_regexp__.match(s)
if m:
s = bb.data.expand(m.group(1), data)
-# debug(2, "CONF %s:%d: including %s" % (fn, lineno, s))
- include(fn, s, data)
+ bb.msg.debug(3, bb.msg.domain.Parsing, "CONF %s:%d: including %s" % (fn, lineno, s))
+ include(fn, s, data, False)
return
m = __require_regexp__.match(s)
if m:
s = bb.data.expand(m.group(1), data)
- include(fn, s, data, True)
+ include(fn, s, data, "include required")
return
raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));
diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py
new file mode 100644
index 000000000..3cb7cc1f0
--- /dev/null
+++ b/bitbake/lib/bb/providers.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003, 2004 Chris Larson
+# Copyright (C) 2003, 2004 Phil Blundell
+# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
+# Copyright (C) 2005 Holger Hans Peter Freyther
+# Copyright (C) 2005 ROAD GmbH
+# Copyright (C) 2006 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA 02111-1307 USA.
+
+import os, re
+from bb import data, utils
+import bb
+
+class NoProvider(Exception):
+ """Exception raised when no provider of a build dependency can be found"""
+
+class NoRProvider(Exception):
+ """Exception raised when no provider of a runtime dependency can be found"""
+
+def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
+ """
+ If there is a PREFERRED_VERSION, find the highest-priority bbfile
+ providing that version. If not, find the latest version provided by
+ an bbfile in the highest-priority set.
+ """
+ if not pkg_pn:
+ pkg_pn = dataCache.pkg_pn
+
+ files = pkg_pn[pn]
+ priorities = {}
+ for f in files:
+ priority = dataCache.bbfile_priority[f]
+ if priority not in priorities:
+ priorities[priority] = []
+ priorities[priority].append(f)
+ p_list = priorities.keys()
+ p_list.sort(lambda a, b: a - b)
+ tmp_pn = []
+ for p in p_list:
+ tmp_pn = [priorities[p]] + tmp_pn
+
+ preferred_file = None
+
+ localdata = data.createCopy(cfgData)
+ bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
+ bb.data.update_data(localdata)
+
+ preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
+ if preferred_v:
+ m = re.match('(.*)_(.*)', preferred_v)
+ if m:
+ preferred_v = m.group(1)
+ preferred_r = m.group(2)
+ else:
+ preferred_r = None
+
+ for file_set in tmp_pn:
+ for f in file_set:
+ pv,pr = dataCache.pkg_pvpr[f]
+ if preferred_v == pv and (preferred_r == pr or preferred_r == None):
+ preferred_file = f
+ preferred_ver = (pv, pr)
+ break
+ if preferred_file:
+ break;
+ if preferred_r:
+ pv_str = '%s-%s' % (preferred_v, preferred_r)
+ else:
+ pv_str = preferred_v
+ itemstr = ""
+ if item:
+ itemstr = " (for item %s)" % item
+ if preferred_file is None:
+ bb.msg.note(1, bb.msg.domain.Provider, "preferred version %s of %s not available%s" % (pv_str, pn, itemstr))
+ else:
+ bb.msg.debug(1, bb.msg.domain.Provider, "selecting %s as PREFERRED_VERSION %s of package %s%s" % (preferred_file, pv_str, pn, itemstr))
+
+ del localdata
+
+ # get highest priority file set
+ files = tmp_pn[0]
+ latest = None
+ latest_p = 0
+ latest_f = None
+ for file_name in files:
+ pv,pr = dataCache.pkg_pvpr[file_name]
+ dp = dataCache.pkg_dp[file_name]
+
+ if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
+ latest = (pv, pr)
+ latest_f = file_name
+ latest_p = dp
+ if preferred_file is None:
+ preferred_file = latest_f
+ preferred_ver = latest
+
+ return (latest,latest_f,preferred_ver, preferred_file)
+
+#
+# RP - build_cache_fail needs to move elsewhere
+#
+def filterProviders(providers, item, cfgData, dataCache, build_cache_fail = {}):
+ """
+ Take a list of providers and filter/reorder according to the
+ environment variables and previous build results
+ """
+ eligible = []
+ preferred_versions = {}
+
+ # Collate providers by PN
+ pkg_pn = {}
+ for p in providers:
+ pn = dataCache.pkg_fn[p]
+ if pn not in pkg_pn:
+ pkg_pn[pn] = []
+ pkg_pn[pn].append(p)
+
+ bb.msg.debug(1, bb.msg.domain.Provider, "providers for %s are: %s" % (item, pkg_pn.keys()))
+
+ for pn in pkg_pn.keys():
+ preferred_versions[pn] = bb.providers.findBestProvider(pn, cfgData, dataCache, pkg_pn, item)[2:4]
+ eligible.append(preferred_versions[pn][1])
+
+
+ for p in eligible:
+ if p in build_cache_fail:
+ bb.msg.debug(1, bb.msg.domain.Provider, "rejecting already-failed %s" % p)
+ eligible.remove(p)
+
+ if len(eligible) == 0:
+ bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item)
+ return 0
+
+
+ # If pn == item, give it a slight default preference
+ # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
+ for p in providers:
+ pn = dataCache.pkg_fn[p]
+ if pn != item:
+ continue
+ (newvers, fn) = preferred_versions[pn]
+ if not fn in eligible:
+ continue
+ eligible.remove(fn)
+ eligible = [fn] + eligible
+
+ # look to see if one of them is already staged, or marked as preferred.
+ # if so, bump it to the head of the queue
+ for p in providers:
+ pn = dataCache.pkg_fn[p]
+ pv, pr = dataCache.pkg_pvpr[p]
+
+ stamp = '%s.do_populate_staging' % dataCache.stamp[p]
+ if os.path.exists(stamp):
+ (newvers, fn) = preferred_versions[pn]
+ if not fn in eligible:
+ # package was made ineligible by already-failed check
+ continue
+ oldver = "%s-%s" % (pv, pr)
+ newver = '-'.join(newvers)
+ if (newver != oldver):
+ extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
+ else:
+ extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
+
+ bb.msg.note(2, bb.msg.domain.Provider, "%s" % extra_chat)
+ eligible.remove(fn)
+ eligible = [fn] + eligible
+ break
+
+ return eligible
+
+def getRuntimeProviders(dataCache, rdepend):
+ """
+ Return any providers of runtime dependency
+ """
+ rproviders = []
+
+ if rdepend in dataCache.rproviders:
+ rproviders += dataCache.rproviders[rdepend]
+
+ if rdepend in dataCache.packages:
+ rproviders += dataCache.packages[rdepend]
+
+ if rproviders:
+ return rproviders
+
+ # Only search dynamic packages if we can't find anything in other variables
+ for pattern in dataCache.packages_dynamic:
+ regexp = re.compile(pattern)
+ if regexp.match(rdepend):
+ rproviders += dataCache.packages_dynamic[pattern]
+
+ return rproviders
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
new file mode 100644
index 000000000..3dde9a9ff
--- /dev/null
+++ b/bitbake/lib/bb/runqueue.py
@@ -0,0 +1,491 @@
+ #!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'RunQueue' implementation
+
+Handles preparation and execution of a queue of tasks
+
+Copyright (C) 2006 Richard Purdie
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License version 2 as published by the Free
+Software Foundation
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+"""
+
+from bb import msg, data, fetch, event, mkdirhier, utils
+from sets import Set
+import bb, os, sys
+
+class TaskFailure(Exception):
+ """Exception raised when a task in a runqueue fails"""
+
+ def __init__(self, fnid, fn, taskname):
+ self.args = fnid, fn, taskname
+
+class RunQueue:
+ """
+ BitBake Run Queue implementation
+ """
+ def __init__(self):
+ self.reset_runqueue()
+
+ def reset_runqueue(self):
+ self.runq_fnid = []
+ self.runq_task = []
+ self.runq_depends = []
+ self.runq_revdeps = []
+ self.runq_weight = []
+ self.prio_map = []
+
+ def get_user_idstring(self, task, taskData):
+ fn = taskData.fn_index[self.runq_fnid[task]]
+ taskname = self.runq_task[task]
+ return "%s, %s" % (fn, taskname)
+
+ def prepare_runqueue(self, cfgData, dataCache, taskData, targets):
+ """
+ Turn a set of taskData into a RunQueue and compute data needed
+ to optimise the execution order.
+ targets is list of paired values - a provider name and the task to run
+ """
+
+ depends = []
+ runq_weight1 = []
+ runq_build = []
+ runq_done = []
+
+ bb.msg.note(1, bb.msg.domain.RunQueue, "Preparing Runqueue")
+
+ for task in range(len(taskData.tasks_name)):
+ fnid = taskData.tasks_fnid[task]
+ fn = taskData.fn_index[fnid]
+ task_deps = dataCache.task_deps[fn]
+
+ if fnid not in taskData.failed_fnids:
+
+ depends = taskData.tasks_tdepends[task]
+
+ # Resolve Depends
+ if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']:
+ taskname = task_deps['deptask'][taskData.tasks_name[task]]
+ for depid in taskData.depids[fnid]:
+ if depid in taskData.build_targets:
+ depdata = taskData.build_targets[depid][0]
+ if depdata:
+ dep = taskData.fn_index[depdata]
+ depends.append(taskData.gettask_id(dep, taskname))
+
+ # Resolve Runtime Depends
+ if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']:
+ taskname = task_deps['rdeptask'][taskData.tasks_name[task]]
+ for depid in taskData.rdepids[fnid]:
+ if depid in taskData.run_targets:
+ depdata = taskData.run_targets[depid][0]
+ if depdata:
+ dep = taskData.fn_index[depdata]
+ depends.append(taskData.gettask_id(dep, taskname))
+
+ def add_recursive_build(depid):
+ """
+ Add build depends of depid to depends
+ (if we've not see it before)
+ (calls itself recursively)
+ """
+ if str(depid) in dep_seen:
+ return
+ dep_seen.append(depid)
+ if depid in taskData.build_targets:
+ depdata = taskData.build_targets[depid][0]
+ if depdata:
+ dep = taskData.fn_index[depdata]
+ taskid = taskData.gettask_id(dep, taskname)
+ depends.append(taskid)
+ fnid = taskData.tasks_fnid[taskid]
+ for nextdepid in taskData.depids[fnid]:
+ if nextdepid not in dep_seen:
+ add_recursive_build(nextdepid)
+ for nextdepid in taskData.rdepids[fnid]:
+ if nextdepid not in rdep_seen:
+ add_recursive_run(nextdepid)
+
+ def add_recursive_run(rdepid):
+ """
+ Add runtime depends of rdepid to depends
+ (if we've not see it before)
+ (calls itself recursively)
+ """
+ if str(rdepid) in rdep_seen:
+ return
+ rdep_seen.append(rdepid)
+ if rdepid in taskData.run_targets:
+ depdata = taskData.run_targets[rdepid][0]
+ if depdata:
+ dep = taskData.fn_index[depdata]
+ taskid = taskData.gettask_id(dep, taskname)
+ depends.append(taskid)
+ fnid = taskData.tasks_fnid[taskid]
+ for nextdepid in taskData.depids[fnid]:
+ if nextdepid not in dep_seen:
+ add_recursive_build(nextdepid)
+ for nextdepid in taskData.rdepids[fnid]:
+ if nextdepid not in rdep_seen:
+ add_recursive_run(nextdepid)
+
+
+ # Resolve Recursive Runtime Depends
+ # Also includes all Build Depends (and their runtime depends)
+ if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']:
+ dep_seen = []
+ rdep_seen = []
+ taskname = task_deps['recrdeptask'][taskData.tasks_name[task]]
+ for depid in taskData.depids[fnid]:
+ add_recursive_build(depid)
+ for rdepid in taskData.rdepids[fnid]:
+ add_recursive_run(rdepid)
+
+ #Prune self references
+ if task in depends:
+ newdep = []
+ bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends))
+ for dep in depends:
+ if task != dep:
+ newdep.append(dep)
+ depends = newdep
+
+
+ self.runq_fnid.append(taskData.tasks_fnid[task])
+ self.runq_task.append(taskData.tasks_name[task])
+ self.runq_depends.append(Set(depends))
+ self.runq_revdeps.append(Set())
+ self.runq_weight.append(0)
+
+ runq_weight1.append(0)
+ runq_build.append(0)
+ runq_done.append(0)
+
+ bb.msg.note(2, bb.msg.domain.RunQueue, "Marking Active Tasks")
+
+ def mark_active(listid, depth):
+ """
+ Mark an item as active along with its depends
+ (calls itself recursively)
+ """
+
+ if runq_build[listid] == 1:
+ return
+
+ runq_build[listid] = 1
+
+ depends = self.runq_depends[listid]
+ for depend in depends:
+ mark_active(depend, depth+1)
+
+ for target in targets:
+ targetid = taskData.getbuild_id(target[0])
+ if targetid in taskData.failed_deps:
+ continue
+
+ if targetid not in taskData.build_targets:
+ continue
+
+ fnid = taskData.build_targets[targetid][0]
+ if fnid in taskData.failed_fnids:
+ continue
+
+ listid = taskData.tasks_lookup[fnid][target[1]]
+
+ mark_active(listid, 1)
+
+ # Prune inactive tasks
+ maps = []
+ delcount = 0
+ for listid in range(len(self.runq_fnid)):
+ if runq_build[listid-delcount] == 1:
+ maps.append(listid-delcount)
+ else:
+ del self.runq_fnid[listid-delcount]
+ del self.runq_task[listid-delcount]
+ del self.runq_depends[listid-delcount]
+ del self.runq_weight[listid-delcount]
+ del runq_weight1[listid-delcount]
+ del runq_build[listid-delcount]
+ del runq_done[listid-delcount]
+ del self.runq_revdeps[listid-delcount]
+ delcount = delcount + 1
+ maps.append(-1)
+
+ if len(self.runq_fnid) == 0:
+ if not taskData.abort:
+ bb.msg.note(1, bb.msg.domain.RunQueue, "All possible tasks have been run but build incomplete (--continue mode). See errors above for incomplete tasks.")
+ return
+ bb.msg.fatal(bb.msg.domain.RunQueue, "No active tasks and not in --continue mode?! Please report this bug.")
+
+ bb.msg.note(2, bb.msg.domain.RunQueue, "Pruned %s inactive tasks, %s left" % (delcount, len(self.runq_fnid)))
+
+ for listid in range(len(self.runq_fnid)):
+ newdeps = []
+ origdeps = self.runq_depends[listid]
+ for origdep in origdeps:
+ if maps[origdep] == -1:
+ bb.msg.fatal(bb.msg.domain.RunQueue, "Invalid mapping - Should never happen!")
+ newdeps.append(maps[origdep])
+ self.runq_depends[listid] = Set(newdeps)
+
+ bb.msg.note(2, bb.msg.domain.RunQueue, "Assign Weightings")
+
+ for listid in range(len(self.runq_fnid)):
+ for dep in self.runq_depends[listid]:
+ self.runq_revdeps[dep].add(listid)
+
+ endpoints = []
+ for listid in range(len(self.runq_fnid)):
+ revdeps = self.runq_revdeps[listid]
+ if len(revdeps) == 0:
+ runq_done[listid] = 1
+ self.runq_weight[listid] = 1
+ endpoints.append(listid)
+ for dep in revdeps:
+ if dep in self.runq_depends[listid]:
+ #self.dump_data(taskData)
+ bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep] , taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid]))
+ runq_weight1[listid] = len(revdeps)
+
+ bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints))
+
+ while 1:
+ next_points = []
+ for listid in endpoints:
+ for revdep in self.runq_depends[listid]:
+ self.runq_weight[revdep] = self.runq_weight[revdep] + self.runq_weight[listid]
+ runq_weight1[revdep] = runq_weight1[revdep] - 1
+ if runq_weight1[revdep] == 0:
+ next_points.append(revdep)
+ runq_done[revdep] = 1
+ endpoints = next_points
+ if len(next_points) == 0:
+ break
+
+ # Sanity Checks
+ for task in range(len(self.runq_fnid)):
+ if runq_done[task] == 0:
+ seen = []
+ deps_seen = []
+ def print_chain(taskid, finish):
+ seen.append(taskid)
+ for revdep in self.runq_revdeps[taskid]:
+ if runq_done[revdep] == 0 and revdep not in seen and not finish:
+ bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) (depends: %s)" % (revdep, self.get_user_idstring(revdep, taskData), self.runq_depends[revdep]))
+ if revdep in deps_seen:
+ bb.msg.error(bb.msg.domain.RunQueue, "Chain ends at Task %s (%s)" % (revdep, self.get_user_idstring(revdep, taskData)))
+ finish = True
+ return
+ for dep in self.runq_depends[revdep]:
+ deps_seen.append(dep)
+ print_chain(revdep, finish)
+ print_chain(task, False)
+ bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) not processed!\nThis is probably a circular dependency (the chain might be printed above)." % (task, self.get_user_idstring(task, taskData)))
+ if runq_weight1[task] != 0:
+ bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) count not zero!" % (task, self.get_user_idstring(task, taskData)))
+
+ # Make a weight sorted map
+ from copy import deepcopy
+
+ sortweight = deepcopy(self.runq_weight)
+ sortweight.sort()
+ copyweight = deepcopy(self.runq_weight)
+ self.prio_map = []
+
+ for weight in sortweight:
+ idx = copyweight.index(weight)
+ self.prio_map.append(idx)
+ copyweight[idx] = -1
+ self.prio_map.reverse()
+
+ #self.dump_data(taskData)
+
+ def execute_runqueue(self, cooker, cfgData, dataCache, taskData, runlist):
+ """
+ Run the tasks in a queue prepared by prepare_runqueue
+ Upon failure, optionally try to recover the build using any alternate providers
+ (if the abort on failure configuration option isn't set)
+ """
+
+ failures = 0
+ while 1:
+ try:
+ self.execute_runqueue_internal(cooker, cfgData, dataCache, taskData)
+ return failures
+ except bb.runqueue.TaskFailure, (fnid, taskData.fn_index[fnid], taskname):
+ if taskData.abort:
+ raise
+ taskData.fail_fnid(fnid)
+ self.reset_runqueue()
+ self.prepare_runqueue(cfgData, dataCache, taskData, runlist)
+ failures = failures + 1
+
+ def execute_runqueue_internal(self, cooker, cfgData, dataCache, taskData):
+ """
+ Run the tasks in a queue prepared by prepare_runqueue
+ """
+
+ bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
+
+ runq_buildable = []
+ runq_running = []
+ runq_complete = []
+ active_builds = 0
+ build_pids = {}
+
+ if len(self.runq_fnid) == 0:
+ # nothing to do
+ return
+
+ def get_next_task(data):
+ """
+ Return the id of the highest priority task that is buildable
+ """
+ for task1 in range(len(data.runq_fnid)):
+ task = data.prio_map[task1]
+ if runq_running[task] == 1:
+ continue
+ if runq_buildable[task] == 1:
+ return task
+ return None
+
+ def task_complete(data, task):
+ """
+ Mark a task as completed
+ Look at the reverse dependencies and mark any task with
+ completed dependencies as buildable
+ """
+ runq_complete[task] = 1
+ for revdep in data.runq_revdeps[task]:
+ if runq_running[revdep] == 1:
+ continue
+ if runq_buildable[revdep] == 1:
+ continue
+ alldeps = 1
+ for dep in data.runq_depends[revdep]:
+ if runq_complete[dep] != 1:
+ alldeps = 0
+ if alldeps == 1:
+ runq_buildable[revdep] = 1
+ fn = taskData.fn_index[self.runq_fnid[revdep]]
+ taskname = self.runq_task[revdep]
+ bb.msg.debug(1, bb.msg.domain.RunQueue, "Marking task %s (%s, %s) as buildable" % (revdep, fn, taskname))
+
+ # Mark initial buildable tasks
+ for task in range(len(self.runq_fnid)):
+ runq_running.append(0)
+ runq_complete.append(0)
+ if len(self.runq_depends[task]) == 0:
+ runq_buildable.append(1)
+ else:
+ runq_buildable.append(0)
+
+
+ number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
+
+ try:
+ while 1:
+ task = get_next_task(self)
+ if task is not None:
+ fn = taskData.fn_index[self.runq_fnid[task]]
+ taskname = self.runq_task[task]
+
+ if bb.build.stamp_is_current_cache(dataCache, fn, taskname):
+ targetid = taskData.gettask_id(fn, taskname)
+ if not (targetid in taskData.external_targets and cooker.configuration.force):
+ bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task, taskData)))
+ runq_running[task] = 1
+ task_complete(self, task)
+ continue
+
+ bb.msg.debug(1, bb.msg.domain.RunQueue, "Running task %s (%s)" % (task, self.get_user_idstring(task, taskData)))
+ try:
+ pid = os.fork()
+ except OSError, e:
+ bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
+ if pid == 0:
+ cooker.configuration.cmd = taskname[3:]
+ try:
+ cooker.tryBuild(fn, False)
+ except bb.build.EventException:
+ bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
+ sys.exit(1)
+ except:
+ bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
+ raise
+ sys.exit(0)
+ build_pids[pid] = task
+ runq_running[task] = 1
+ active_builds = active_builds + 1
+ if active_builds < number_tasks:
+ continue
+ if active_builds > 0:
+ result = os.waitpid(-1, 0)
+ active_builds = active_builds - 1
+ task = build_pids[result[0]]
+ if result[1] != 0:
+ bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task, taskData)))
+ raise bb.runqueue.TaskFailure(self.runq_fnid[task], taskData.fn_index[self.runq_fnid[task]], self.runq_task[task])
+ task_complete(self, task)
+ del build_pids[result[0]]
+ continue
+ break
+ except SystemExit:
+ raise
+ except:
+ bb.msg.error(bb.msg.domain.RunQueue, "Exception received")
+ while active_builds > 0:
+ bb.msg.note(1, bb.msg.domain.RunQueue, "Waiting for %s active tasks to finish" % active_builds)
+ tasknum = 1
+ for k, v in build_pids.iteritems():
+ bb.msg.note(1, bb.msg.domain.RunQueue, "%s: %s (%s)" % (tasknum, self.get_user_idstring(v, taskData), k))
+ tasknum = tasknum + 1
+ result = os.waitpid(-1, 0)
+ del build_pids[result[0]]
+ active_builds = active_builds - 1
+ raise
+
+ # Sanity Checks
+ for task in range(len(self.runq_fnid)):
+ if runq_buildable[task] == 0:
+ bb.msg.error(bb.msg.domain.RunQueue, "Task %s never buildable!" % task)
+ if runq_running[task] == 0:
+ bb.msg.error(bb.msg.domain.RunQueue, "Task %s never ran!" % task)
+ if runq_complete[task] == 0:
+ bb.msg.error(bb.msg.domain.RunQueue, "Task %s never completed!" % task)
+
+ return 0
+
+ def dump_data(self, taskQueue):
+ """
+ Dump some debug information on the internal data structures
+ """
+ bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:")
+ for task in range(len(self.runq_fnid)):
+ bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task,
+ taskQueue.fn_index[self.runq_fnid[task]],
+ self.runq_task[task],
+ self.runq_weight[task],
+ self.runq_depends[task],
+ self.runq_revdeps[task]))
+
+ bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:")
+ for task1 in range(len(self.runq_fnid)):
+ if task1 in self.prio_map:
+ task = self.prio_map[task1]
+ bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task,
+ taskQueue.fn_index[self.runq_fnid[task]],
+ self.runq_task[task],
+ self.runq_weight[task],
+ self.runq_depends[task],
+ self.runq_revdeps[task]))
diff --git a/bitbake/lib/bb/shell.py b/bitbake/lib/bb/shell.py
index 93ad00d1e..760c371d9 100644
--- a/bitbake/lib/bb/shell.py
+++ b/bitbake/lib/bb/shell.py
@@ -56,9 +56,8 @@ try:
set
except NameError:
from sets import Set as set
-import sys, os, imp, readline, socket, httplib, urllib, commands, popen2, copy, shlex, Queue, fnmatch
-imp.load_source( "bitbake", os.path.dirname( sys.argv[0] )+"/bitbake" )
-from bb import data, parse, build, fatal
+import sys, os, readline, socket, httplib, urllib, commands, popen2, copy, shlex, Queue, fnmatch
+from bb import data, parse, build, fatal, cache, taskdata, runqueue, providers as Providers
__version__ = "0.5.3.1"
__credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de>
@@ -108,7 +107,7 @@ class BitBakeShellCommands:
preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 )
if not preferred: preferred = item
try:
- lv, lf, pv, pf = cooker.findBestProvider( preferred )
+ lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status, cooker.build_cache_fail)
except KeyError:
if item in cooker.status.providers:
pf = cooker.status.providers[item][0]
@@ -156,14 +155,39 @@ class BitBakeShellCommands:
cooker.build_cache = []
cooker.build_cache_fail = []
- for name in names:
- try:
- cooker.buildProvider( name, data.getVar("BUILD_ALL_DEPS", cooker.configuration.data, True) )
- except build.EventException, e:
- print "ERROR: Couldn't build '%s'" % name
- global last_exception
- last_exception = e
- break
+ td = taskdata.TaskData(cooker.configuration.abort)
+
+ try:
+ tasks = []
+ for name in names:
+ td.add_provider(cooker.configuration.data, cooker.status, name)
+ providers = td.get_provider(name)
+
+ if len(providers) == 0:
+ raise Providers.NoProvider
+
+ tasks.append([name, "do_%s" % cooker.configuration.cmd])
+
+ td.add_unresolved(cooker.configuration.data, cooker.status)
+
+ rq = runqueue.RunQueue()
+ rq.prepare_runqueue(cooker.configuration.data, cooker.status, td, tasks)
+ rq.execute_runqueue(cooker, cooker.configuration.data, cooker.status, td, tasks)
+
+ except Providers.NoProvider:
+ print "ERROR: No Provider"
+ global last_exception
+ last_exception = Providers.NoProvider
+
+ except runqueue.TaskFailure, (fnid, fn, taskname):
+ print "ERROR: '%s, %s' failed" % (fn, taskname)
+ global last_exception
+ last_exception = runqueue.TaskFailure
+
+ except build.EventException, e:
+ print "ERROR: Couldn't build '%s'" % names
+ global last_exception
+ last_exception = e
cooker.configuration.cmd = oldcmd
@@ -233,7 +257,7 @@ class BitBakeShellCommands:
item = data.getVar('PN', bbfile_data, 1)
data.setVar( "_task_cache", [], bbfile_data ) # force
try:
- cooker.tryBuildPackage( os.path.abspath( bf ), item, bbfile_data )
+ cooker.tryBuildPackage( os.path.abspath( bf ), item, cmd, bbfile_data, True )
except build.EventException, e:
print "ERROR: Couldn't build '%s'" % name
global last_exception
@@ -255,8 +279,7 @@ class BitBakeShellCommands:
def fileRebuild( self, params ):
"""Rebuild (clean & build) a .bb file"""
- self.fileClean( params )
- self.fileBuild( params )
+ self.fileBuild( params, "rebuild" )
fileRebuild.usage = "<bbfile>"
def fileReparse( self, params ):
@@ -265,14 +288,19 @@ class BitBakeShellCommands:
print "SHELL: Parsing '%s'" % bbfile
parse.update_mtime( bbfile )
cooker.bb_cache.cacheValidUpdate(bbfile)
- fromCache = cooker.bb_cache.loadData(bbfile, cooker)
+ fromCache = cooker.bb_cache.loadData(bbfile, cooker.configuration.data)
cooker.bb_cache.sync()
- if False: #from Cache
+ if False: #fromCache:
print "SHELL: File has not been updated, not reparsing"
else:
print "SHELL: Parsed"
fileReparse.usage = "<bbfile>"
+ def abort( self, params ):
+ """Toggle abort task execution flag (see bitbake -k)"""
+ cooker.configuration.abort = not cooker.configuration.abort
+ print "SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort )
+
def force( self, params ):
"""Toggle force task execution flag (see bitbake -f)"""
cooker.configuration.force = not cooker.configuration.force
@@ -365,18 +393,14 @@ SRC_URI = ""
new.usage = "<directory> <filename>"
def pasteBin( self, params ):
- """Send a command + output buffer to http://pastebin.com"""
+ """Send a command + output buffer to the pastebin at http://rafb.net/paste"""
index = params[0]
contents = self._shell.myout.buffer( int( index ) )
- status, error, location = sendToPastebin( contents )
- if status == 302:
- print "SHELL: Pasted to %s" % location
- else:
- print "ERROR: %s %s" % ( status, error )
+ sendToPastebin( "output of " + params[0], contents )
pasteBin.usage = "<index>"
def pasteLog( self, params ):
- """Send the last event exception error log (if there is one) to http://oe.pastebin.com"""
+ """Send the last event exception error log (if there is one) to http://rafb.net/paste"""
if last_exception is None:
print "SHELL: No Errors yet (Phew)..."
else:
@@ -387,12 +411,8 @@ SRC_URI = ""
filename = filename.strip()
print "SHELL: Pasting log file to pastebin..."
- status, error, location = sendToPastebin( open( filename ).read() )
-
- if status == 302:
- print "SHELL: Pasted to %s" % location
- else:
- print "ERROR: %s %s" % ( status, error )
+ file = open( filename ).read()
+ sendToPastebin( "contents of " + filename, file )
def patch( self, params ):
"""Execute 'patch' command on a providee"""
@@ -401,12 +421,13 @@ SRC_URI = ""
def parse( self, params ):
"""(Re-)parse .bb files and calculate the dependency graph"""
- cooker.status = cooker.ParsingStatus()
+ cooker.status = cache.CacheData()
ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or ""
cooker.status.ignored_dependencies = set( ignore.split() )
cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) )
- cooker.collect_bbfiles( cooker.myProgressCallback )
+ (filelist, masked) = cooker.collect_bbfiles()
+ cooker.parse_bbfiles(filelist, masked, cooker.myProgressCallback)
cooker.buildDepgraph()
global parsed
parsed = True
@@ -434,7 +455,7 @@ SRC_URI = ""
name, var = params
bbfile = self._findProvider( name )
if bbfile is not None:
- the_data = cooker.bb_cache.loadDataFull(bbfile, cooker)
+ the_data = cooker.bb_cache.loadDataFull(bbfile, cooker.configuration.data)
value = the_data.getVar( var, 1 )
print value
else:
@@ -538,7 +559,8 @@ SRC_URI = ""
if not preferred: preferred = item
try:
- lv, lf, pv, pf = cooker.findBestProvider( preferred )
+ lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status,
+cooker.build_cache_fail)
except KeyError:
lv, lf, pv, pf = (None,)*4
@@ -565,24 +587,29 @@ def completeFilePath( bbfile ):
return key
return bbfile
-def sendToPastebin( content ):
+def sendToPastebin( desc, content ):
"""Send content to http://oe.pastebin.com"""
mydata = {}
- mydata["parent_pid"] = ""
- mydata["format"] = "bash"
- mydata["code2"] = content
- mydata["paste"] = "Send"
- mydata["poster"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" )
+ mydata["lang"] = "Plain Text"
+ mydata["desc"] = desc
+ mydata["cvt_tabs"] = "No"
+ mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" )
+ mydata["text"] = content
params = urllib.urlencode( mydata )
headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
- conn = httplib.HTTPConnection( "oe.pastebin.com:80" )
- conn.request("POST", "/", params, headers )
+ host = "rafb.net"
+ conn = httplib.HTTPConnection( "%s:80" % host )
+ conn.request("POST", "/paste/paste.php", params, headers )
response = conn.getresponse()
conn.close()
- return response.status, response.reason, response.getheader( "location" ) or "unknown"
+ if response.status == 302:
+ location = response.getheader( "location" ) or "unknown"
+ print "SHELL: Pasted to http://%s%s" % ( host, location )
+ else:
+ print "ERROR: %s %s" % ( response.status, response.reason )
def completer( text, state ):
"""Return a possible readline completion"""
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
new file mode 100644
index 000000000..181bb5e35
--- /dev/null
+++ b/bitbake/lib/bb/taskdata.py
@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'TaskData' implementation
+
+Task data collection and handling
+
+Copyright (C) 2006 Richard Purdie
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License version 2 as published by the Free
+Software Foundation
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+"""
+
+from bb import data, fetch, event, mkdirhier, utils
+import bb, os
+
+class TaskData:
+ """
+ BitBake Task Data implementation
+ """
+ def __init__(self, abort = True):
+ self.build_names_index = []
+ self.run_names_index = []
+ self.fn_index = []
+
+ self.build_targets = {}
+ self.run_targets = {}
+
+ self.external_targets = []
+
+ self.tasks_fnid = []
+ self.tasks_name = []
+ self.tasks_tdepends = []
+ # Cache to speed up task ID lookups
+ self.tasks_lookup = {}
+
+ self.depids = {}
+ self.rdepids = {}
+
+ self.consider_msgs_cache = []
+
+ self.failed_deps = []
+ self.failed_rdeps = []
+ self.failed_fnids = []
+
+ self.abort = abort
+
+ def getbuild_id(self, name):
+ """
+ Return an ID number for the build target name.
+ If it doesn't exist, create one.
+ """
+ if not name in self.build_names_index:
+ self.build_names_index.append(name)
+ return len(self.build_names_index) - 1
+
+ return self.build_names_index.index(name)
+
+ def getrun_id(self, name):
+ """
+ Return an ID number for the run target name.
+ If it doesn't exist, create one.
+ """
+ if not name in self.run_names_index:
+ self.run_names_index.append(name)
+ return len(self.run_names_index) - 1
+
+ return self.run_names_index.index(name)
+
+ def getfn_id(self, name):
+ """
+ Return an ID number for the filename.
+ If it doesn't exist, create one.
+ """
+ if not name in self.fn_index:
+ self.fn_index.append(name)
+ return len(self.fn_index) - 1
+
+ return self.fn_index.index(name)
+
+ def gettask_id(self, fn, task):
+ """
+ Return an ID number for the task matching fn and task.
+ If it doesn't exist, create one.
+ """
+ fnid = self.getfn_id(fn)
+
+ if fnid in self.tasks_lookup:
+ if task in self.tasks_lookup[fnid]:
+ return self.tasks_lookup[fnid][task]
+
+ self.tasks_name.append(task)
+ self.tasks_fnid.append(fnid)
+ self.tasks_tdepends.append([])
+
+ listid = len(self.tasks_name) - 1
+
+ if fnid not in self.tasks_lookup:
+ self.tasks_lookup[fnid] = {}
+ self.tasks_lookup[fnid][task] = listid
+
+ return listid
+
+ def add_tasks(self, fn, dataCache):
+ """
+ Add tasks for a given fn to the database
+ """
+
+ task_graph = dataCache.task_queues[fn]
+ task_deps = dataCache.task_deps[fn]
+
+ fnid = self.getfn_id(fn)
+
+ if fnid in self.failed_fnids:
+ bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
+
+ # Check if we've already seen this fn
+ if fnid in self.tasks_fnid:
+ return
+
+ # Work out task dependencies
+ for task in task_graph.allnodes():
+ parentids = []
+ for dep in task_graph.getparents(task):
+ parentid = self.gettask_id(fn, dep)
+ parentids.append(parentid)
+ taskid = self.gettask_id(fn, task)
+ self.tasks_tdepends[taskid].extend(parentids)
+
+ # Work out build dependencies
+ if not fnid in self.depids:
+ dependids = {}
+ for depend in dataCache.deps[fn]:
+ bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
+ dependids[self.getbuild_id(depend)] = None
+ self.depids[fnid] = dependids.keys()
+
+ # Work out runtime dependencies
+ if not fnid in self.rdepids:
+ rdependids = {}
+ rdepends = dataCache.rundeps[fn]
+ rrecs = dataCache.runrecs[fn]
+ for package in rdepends:
+ for rdepend in rdepends[package]:
+ bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
+ rdependids[self.getrun_id(rdepend)] = None
+ for package in rrecs:
+ for rdepend in rrecs[package]:
+ bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
+ rdependids[self.getrun_id(rdepend)] = None
+ self.rdepids[fnid] = rdependids.keys()
+
+ for dep in self.depids[fnid]:
+ if dep in self.failed_deps:
+ self.fail_fnid(fnid)
+ return
+ for dep in self.rdepids[fnid]:
+ if dep in self.failed_rdeps:
+ self.fail_fnid(fnid)
+ return
+
+ def have_build_target(self, target):
+ """
+ Have we a build target matching this name?
+ """
+ targetid = self.getbuild_id(target)
+
+ if targetid in self.build_targets:
+ return True
+ return False
+
+ def have_runtime_target(self, target):
+ """
+ Have we a runtime target matching this name?
+ """
+ targetid = self.getrun_id(target)
+
+ if targetid in self.run_targets:
+ return True
+ return False
+
+ def add_build_target(self, fn, item):
+ """
+ Add a build target.
+ If already present, append the provider fn to the list
+ """
+ targetid = self.getbuild_id(item)
+ fnid = self.getfn_id(fn)
+
+ if targetid in self.build_targets:
+ if fnid in self.build_targets[targetid]:
+ return
+ self.build_targets[targetid].append(fnid)
+ return
+ self.build_targets[targetid] = [fnid]
+
+ def add_runtime_target(self, fn, item):
+ """
+ Add a runtime target.
+ If already present, append the provider fn to the list
+ """
+ targetid = self.getrun_id(item)
+ fnid = self.getfn_id(fn)
+
+ if targetid in self.run_targets:
+ if fnid in self.run_targets[targetid]:
+ return
+ self.run_targets[targetid].append(fnid)
+ return
+ self.run_targets[targetid] = [fnid]
+
+ def mark_external_target(self, item):
+ """
+ Mark a build target as being externally requested
+ """
+ targetid = self.getbuild_id(item)
+
+ if targetid not in self.external_targets:
+ self.external_targets.append(targetid)
+
+ def get_unresolved_build_targets(self, dataCache):
+ """
+ Return a list of build targets who's providers
+ are unknown.
+ """
+ unresolved = []
+ for target in self.build_names_index:
+ if target in dataCache.ignored_dependencies:
+ continue
+ if self.build_names_index.index(target) in self.failed_deps:
+ continue
+ if not self.have_build_target(target):
+ unresolved.append(target)
+ return unresolved
+
+ def get_unresolved_run_targets(self, dataCache):
+ """
+ Return a list of runtime targets who's providers
+ are unknown.
+ """
+ unresolved = []
+ for target in self.run_names_index:
+ if target in dataCache.ignored_dependencies:
+ continue
+ if self.run_names_index.index(target) in self.failed_rdeps:
+ continue
+ if not self.have_runtime_target(target):
+ unresolved.append(target)
+ return unresolved
+
+ def get_provider(self, item):
+ """
+ Return a list of providers of item
+ """
+ targetid = self.getbuild_id(item)
+
+ return self.build_targets[targetid]
+
+ def get_dependees(self, itemid):
+ """
+ Return a list of targets which depend on item
+ """
+ dependees = []
+ for fnid in self.depids:
+ if itemid in self.depids[fnid]:
+ dependees.append(fnid)
+ return dependees
+
+ def get_dependees_str(self, item):
+ """
+ Return a list of targets which depend on item as a user readable string
+ """
+ itemid = self.getbuild_id(item)
+ dependees = []
+ for fnid in self.depids:
+ if itemid in self.depids[fnid]:
+ dependees.append(self.fn_index[fnid])
+ return dependees
+
+ def get_rdependees(self, itemid):
+ """
+ Return a list of targets which depend on runtime item
+ """
+ dependees = []
+ for fnid in self.rdepids:
+ if itemid in self.rdepids[fnid]:
+ dependees.append(fnid)
+ return dependees
+
+ def get_rdependees_str(self, item):
+ """
+ Return a list of targets which depend on runtime item as a user readable string
+ """
+ itemid = self.getrun_id(item)
+ dependees = []
+ for fnid in self.rdepids:
+ if itemid in self.rdepids[fnid]:
+ dependees.append(self.fn_index[fnid])
+ return dependees
+
+ def add_provider(self, cfgData, dataCache, item):
+ try:
+ self.add_provider_internal(cfgData, dataCache, item)
+ except bb.providers.NoProvider:
+ if self.abort:
+ bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
+ raise
+ targetid = self.getbuild_id(item)
+ self.remove_buildtarget(targetid)
+
+ self.mark_external_target(item)
+
+ def add_provider_internal(self, cfgData, dataCache, item):
+ """
+ Add the providers of item to the task data
+ Mark entries were specifically added externally as against dependencies
+ added internally during dependency resolution
+ """
+
+ if item in dataCache.ignored_dependencies:
+ return
+
+ if not item in dataCache.providers:
+ bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
+ bb.event.fire(bb.event.NoProvider(item, cfgData))
+ raise bb.providers.NoProvider(item)
+
+ if self.have_build_target(item):
+ return
+
+ all_p = dataCache.providers[item]
+
+ eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
+
+ for p in eligible:
+ fnid = self.getfn_id(p)
+ if fnid in self.failed_fnids:
+ eligible.remove(p)
+
+ if not eligible:
+ bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
+ bb.event.fire(bb.event.NoProvider(item, cfgData))
+ raise bb.providers.NoProvider(item)
+
+ prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
+ if prefervar:
+ dataCache.preferred[item] = prefervar
+
+ discriminated = False
+ if item in dataCache.preferred:
+ for p in eligible:
+ pn = dataCache.pkg_fn[p]
+ if dataCache.preferred[item] == pn:
+ bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
+ eligible.remove(p)
+ eligible = [p] + eligible
+ discriminated = True
+ break
+
+ if len(eligible) > 1 and discriminated == False:
+ if item not in self.consider_msgs_cache:
+ providers_list = []
+ for fn in eligible:
+ providers_list.append(dataCache.pkg_fn[fn])
+ bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
+ bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
+ bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
+ self.consider_msgs_cache.append(item)
+
+ for fn in eligible:
+ fnid = self.getfn_id(fn)
+ if fnid in self.failed_fnids:
+ continue
+ bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
+ self.add_build_target(fn, item)
+ self.add_tasks(fn, dataCache)
+
+
+ #item = dataCache.pkg_fn[fn]
+
+ def add_rprovider(self, cfgData, dataCache, item):
+ """
+ Add the runtime providers of item to the task data
+ (takes item names from RDEPENDS/PACKAGES namespace)
+ """
+
+ if item in dataCache.ignored_dependencies:
+ return
+
+ if self.have_runtime_target(item):
+ return
+
+ all_p = bb.providers.getRuntimeProviders(dataCache, item)
+
+ if not all_p:
+ bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
+ bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
+ raise bb.providers.NoRProvider(item)
+
+ eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
+
+ for p in eligible:
+ fnid = self.getfn_id(p)
+ if fnid in self.failed_fnids:
+ eligible.remove(p)
+
+ if not eligible:
+ bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
+ bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
+ raise bb.providers.NoRProvider(item)
+
+ # Should use dataCache.preferred here?
+ preferred = []
+ for p in eligible:
+ pn = dataCache.pkg_fn[p]
+ provides = dataCache.pn_provides[pn]
+ for provide in provides:
+ prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
+ if prefervar == pn:
+ bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
+ eligible.remove(p)
+ eligible = [p] + eligible
+ preferred.append(p)
+
+ if len(eligible) > 1 and len(preferred) == 0:
+ if item not in self.consider_msgs_cache:
+ providers_list = []
+ for fn in eligible:
+ providers_list.append(dataCache.pkg_fn[fn])
+ bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
+ bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
+ bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
+ self.consider_msgs_cache.append(item)
+
+ if len(preferred) > 1:
+ if item not in self.consider_msgs_cache:
+ providers_list = []
+ for fn in preferred:
+ providers_list.append(dataCache.pkg_fn[fn])
+ bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
+ bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
+ bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
+ self.consider_msgs_cache.append(item)
+
+ # run through the list until we find one that we can build
+ for fn in eligible:
+ fnid = self.getfn_id(fn)
+ if fnid in self.failed_fnids:
+ continue
+ bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
+ self.add_runtime_target(fn, item)
+ self.add_tasks(fn, dataCache)
+
+ def fail_fnid(self, fnid):
+ """
+ Mark a file as failed (unbuildable)
+ Remove any references from build and runtime provider lists
+ """
+ if fnid in self.failed_fnids:
+ return
+ bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
+ self.failed_fnids.append(fnid)
+ for target in self.build_targets:
+ if fnid in self.build_targets[target]:
+ self.build_targets[target].remove(fnid)
+ if len(self.build_targets[target]) == 0:
+ self.remove_buildtarget(target)
+ for target in self.run_targets:
+ if fnid in self.run_targets[target]:
+ self.run_targets[target].remove(fnid)
+ if len(self.run_targets[target]) == 0:
+ self.remove_runtarget(target)
+
+ def remove_buildtarget(self, targetid):
+ """
+ Mark a build target as failed (unbuildable)
+ Trigger removal of any files that have this as a dependency
+ """
+ bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
+ self.failed_deps.append(targetid)
+ dependees = self.get_dependees(targetid)
+ for fnid in dependees:
+ self.fail_fnid(fnid)
+ if self.abort and targetid in self.external_targets:
+ bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid])
+ raise bb.providers.NoProvider
+
+ def remove_runtarget(self, targetid):
+ """
+ Mark a run target as failed (unbuildable)
+ Trigger removal of any files that have this as a dependency
+ """
+ bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
+ self.failed_rdeps.append(targetid)
+ dependees = self.get_rdependees(targetid)
+ for fnid in dependees:
+ self.fail_fnid(fnid)
+
+ def add_unresolved(self, cfgData, dataCache):
+ """
+ Resolve all unresolved build and runtime targets
+ """
+ bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
+ while 1:
+ added = 0
+ for target in self.get_unresolved_build_targets(dataCache):
+ try:
+ self.add_provider_internal(cfgData, dataCache, target)
+ added = added + 1
+ except bb.providers.NoProvider:
+ targetid = self.getbuild_id(target)
+ if self.abort and targetid in self.external_targets:
+ raise
+ self.remove_buildtarget(targetid)
+ for target in self.get_unresolved_run_targets(dataCache):
+ try:
+ self.add_rprovider(cfgData, dataCache, target)
+ added = added + 1
+ except bb.providers.NoRProvider:
+ self.remove_runtarget(self.getrun_id(target))
+ bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
+ if added == 0:
+ break
+
+ def dump_data(self):
+ """
+ Dump some debug information on the internal data structures
+ """
+ bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
+ bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
+ bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
+ bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
+ bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
+ for target in self.build_targets.keys():
+ bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
+ bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
+ for target in self.run_targets.keys():
+ bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
+ bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
+ for task in range(len(self.tasks_name)):
+ bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
+ task,
+ self.fn_index[self.tasks_fnid[task]],
+ self.tasks_name[task],
+ self.tasks_tdepends[task]))
+ bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
+ for fnid in self.rdepids:
+ bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
+
+
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 5b3cb38d8..d7383f44b 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -103,11 +103,11 @@ def _print_trace(body, line):
import bb
# print the environment of the method
- bb.error("Printing the environment of the function")
+ bb.msg.error(bb.msg.domain.Util, "Printing the environment of the function")
min_line = max(1,line-4)
max_line = min(line+4,len(body)-1)
for i in range(min_line,max_line+1):
- bb.error("\t%.4d:%s" % (i, body[i-1]) )
+ bb.msg.error(bb.msg.domain.Util, "\t%.4d:%s" % (i, body[i-1]) )
def better_compile(text, file, realfile):
@@ -122,9 +122,9 @@ def better_compile(text, file, realfile):
# split the text into lines again
body = text.split('\n')
- bb.error("Error in compiling: ", realfile)
- bb.error("The lines resulting into this error were:")
- bb.error("\t%d:%s:'%s'" % (e.lineno, e.__class__.__name__, body[e.lineno-1]))
+ bb.msg.error(bb.msg.domain.Util, "Error in compiling: ", realfile)
+ bb.msg.error(bb.msg.domain.Util, "The lines resulting into this error were:")
+ bb.msg.error(bb.msg.domain.Util, "\t%d:%s:'%s'" % (e.lineno, e.__class__.__name__, body[e.lineno-1]))
_print_trace(body, e.lineno)
@@ -147,8 +147,8 @@ def better_exec(code, context, text, realfile):
raise
# print the Header of the Error Message
- bb.error("Error in executing: ", realfile)
- bb.error("Exception:%s Message:%s" % (t,value) )
+ bb.msg.error(bb.msg.domain.Util, "Error in executing: ", realfile)
+ bb.msg.error(bb.msg.domain.Util, "Exception:%s Message:%s" % (t,value) )
# let us find the line number now
while tb.tb_next:
@@ -160,3 +160,43 @@ def better_exec(code, context, text, realfile):
_print_trace( text.split('\n'), line )
raise
+
+def Enum(*names):
+ """
+ A simple class to give Enum support
+ """
+
+ assert names, "Empty enums are not supported"
+
+ class EnumClass(object):
+ __slots__ = names
+ def __iter__(self): return iter(constants)
+ def __len__(self): return len(constants)
+ def __getitem__(self, i): return constants[i]
+ def __repr__(self): return 'Enum' + str(names)
+ def __str__(self): return 'enum ' + str(constants)
+
+ class EnumValue(object):
+ __slots__ = ('__value')
+ def __init__(self, value): self.__value = value
+ Value = property(lambda self: self.__value)
+ EnumType = property(lambda self: EnumType)
+ def __hash__(self): return hash(self.__value)
+ def __cmp__(self, other):
+ # C fans might want to remove the following assertion
+ # to make all enums comparable by ordinal value {;))
+ assert self.EnumType is other.EnumType, "Only values from the same enum are comparable"
+ return cmp(self.__value, other.__value)
+ def __invert__(self): return constants[maximum - self.__value]
+ def __nonzero__(self): return bool(self.__value)
+ def __repr__(self): return str(names[self.__value])
+
+ maximum = len(names) - 1
+ constants = [None] * len(names)
+ for i, each in enumerate(names):
+ val = EnumValue(i)
+ setattr(EnumClass, each, val)
+ constants[i] = val
+ constants = tuple(constants)
+ EnumType = EnumClass()
+ return EnumType