bitbake: [parser] Remove the "data" from feeder, evaluate after parsing a file

Evaluate the statements after having parsed one file. This is
referred to as "entwirren" and we can remove the direct evaluation
and postpone a bit, in the future we can use a cached copy instead
of parsing the original.

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

4 files changed, 51 insertions, 64 deletions

diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index b1308b3b0..6737e061e 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -68,15 +68,11 @@ def supports(fn, data):
             return 1
     return 0
 
-def handle(fn, data, include = 0, statements = None):
+def handle(fn, data, include = 0):
     """Call the handler that is appropriate for this file"""
-    if not statements:
-        import ast
-        statements = ast.StatementGroup()
-
     for h in handlers:
         if h['supports'](fn, data):
-            return h['handle'](fn, data, include, statements)
+            return h['handle'](fn, data, include)
     raise ParseError("%s is not a BitBake file" % fn)
 
 def init(fn, data):
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index ed9c689af..646a5b8d8 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -56,13 +56,11 @@ class IncludeNode:
         s = bb.data.expand(self.what_file, data)
         bb.msg.debug(3, bb.msg.domain.Parsing, "CONF %s:%d: including %s" % (self.from_fn, self.from_lineno, s))
 
-        # TODO: Cache those includes...
-        statements = StatementGroup()
+        # TODO: Cache those includes... maybe not here though
         if self.force:
-            bb.parse.ConfHandler.include(statements, self.from_fn, s, data, "include required")
+            bb.parse.ConfHandler.include(self.from_fn, s, data, "include required")
         else:
-            bb.parse.ConfHandler.include(statements, self.from_fn, s, data, False)
-        #statements.eval(data)
+            bb.parse.ConfHandler.include(self.from_fn, s, data, False)
 
 class ExportNode:
     def __init__(self, var):
@@ -256,43 +254,30 @@ class InheritNode:
         self.n = __word__.findall(files)
 
     def eval(self, data):
-        statements = StatementGroup()
-        bb.parse.BBHandler.inherit(statements, self.n, data)
+        bb.parse.BBHandler.inherit(self.n, data)
  
-def handleInclude(statements, m, fn, lineno, data, force):
-    # AST handling
+def handleInclude(statements, m, fn, lineno, force):
     statements.append(IncludeNode(m.group(1), fn, lineno, force))
-    statements[-1].eval(data)
 
-def handleExport(statements, m, data):
-    # AST handling
+def handleExport(statements, m):
     statements.append(ExportNode(m.group(1)))
-    statements[-1].eval(data)
 
-def handleData(statements, groupd, data):
-    # AST handling
+def handleData(statements, groupd):
     statements.append(DataNode(groupd))
-    statements[-1].eval(data)
 
-def handleMethod(statements, func_name, lineno, fn, body, d):
-    # AST handling
+def handleMethod(statements, func_name, lineno, fn, body):
     statements.append(MethodNode(func_name, body, lineno, fn))
-    statements[-1].eval(d)
 
 def handlePythonMethod(statements, root, body, fn):
-    # AST handling
     statements.append(PythonMethodNode(root, body, fn))
-    statements[-1].eval(None)
 
-def handleMethodFlags(statements, key, m, d):
+def handleMethodFlags(statements, key, m):
     statements.append(MethodFlagsNode(key, m))
-    statements[-1].eval(d)
 
-def handleExportFuncs(statements, m, classes, d):
+def handleExportFuncs(statements, m, classes):
     statements.append(ExportFuncsNode(m.group(1), classes))
-    statements[-1].eval(d)
 
-def handleAddTask(statements, m, d):
+def handleAddTask(statements, m):
     func = m.group("func")
     before = m.group("before")
     after = m.group("after")
@@ -300,17 +285,14 @@ def handleAddTask(statements, m, d):
         return
 
     statements.append(AddTaskNode(func, before, after))
-    statements[-1].eval(d)
 
-def handleBBHandlers(statements, m, d):
+def handleBBHandlers(statements, m):
     statements.append(BBHandlerNode(m.group(1)))
-    statements[-1].eval(d)
 
-def handleInherit(statements, m, d):
+def handleInherit(statements, m):
     files = m.group(1)
     n = __word__.findall(files)
     statements.append(InheritNode(m.group(1)))
-    statements[-1].eval(d)
 
 def finalise(fn, d):
     bb.data.expandKeys(d)
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index 32e0901e2..f313009ab 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -62,7 +62,7 @@ IN_PYTHON_EOF = -9999999999999
 def supports(fn, d):
     return fn[-3:] == ".bb" or fn[-8:] == ".bbclass" or fn[-4:] == ".inc"
 
-def inherit(statements, files, d):
+def inherit(files, d):
     __inherit_cache = data.getVar('__inherit_cache', d) or []
     fn = ""
     lineno = 0
@@ -75,10 +75,10 @@ def inherit(statements, files, d):
             bb.msg.debug(2, bb.msg.domain.Parsing, "BB %s:%d: inheriting %s" % (fn, lineno, file))
             __inherit_cache.append( file )
             data.setVar('__inherit_cache', __inherit_cache, d)
-            include(statements, fn, file, d, "inherit")
+            include(fn, file, d, "inherit")
             __inherit_cache = data.getVar('__inherit_cache', d) or []
 
-def handle(fn, d, include, statements):
+def handle(fn, d, include):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__
     __body__ = []
     __infunc__ = ""
@@ -113,19 +113,24 @@ def handle(fn, d, include, statements):
     if include:
         bb.parse.mark_dependency(d, abs_fn)
 
-    if ext != ".bbclass":
-        data.setVar('FILE', fn, d)
-
+    statements = ast.StatementGroup()
     lineno = 0
     while 1:
         lineno = lineno + 1
         s = f.readline()
         if not s: break
         s = s.rstrip()
-        feeder(lineno, s, fn, base_name, d, statements)
+        feeder(lineno, s, fn, base_name, statements)
     if __inpython__:
         # add a blank line to close out any python definition
-        feeder(IN_PYTHON_EOF, "", fn, base_name, d, statements)
+        feeder(IN_PYTHON_EOF, "", fn, base_name, statements)
+
+    # DONE WITH PARSING... time to evaluate
+    if ext != ".bbclass":
+        data.setVar('FILE', fn, d)
+
+    statements.eval(d)
+
     if ext == ".bbclass":
         classes.remove(__classname__)
     else:
@@ -145,7 +150,7 @@ def handle(fn, d, include, statements):
                 pn = data.getVar('PN', d, True)
                 based = bb.data.createCopy(d)
                 data.setVar('PN', pn + '-' + cls, based)
-                inherit(statements, [cls], based)
+                inherit([cls], based)
                 try:
                     ast.finalise(fn, based)
                 except bb.parse.SkipPackage:
@@ -162,12 +167,12 @@ def handle(fn, d, include, statements):
 
     return d
 
-def feeder(lineno, s, fn, root, d, statements):
+def feeder(lineno, s, fn, root, statements):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__,__infunc__, __body__, classes, bb, __residue__
     if __infunc__:
         if s == '}':
             __body__.append('')
-            ast.handleMethod(statements, __infunc__, lineno, fn, __body__, d)
+            ast.handleMethod(statements, __infunc__, lineno, fn, __body__)
             __infunc__ = ""
             __body__ = []
         else:
@@ -201,7 +206,7 @@ def feeder(lineno, s, fn, root, d, statements):
     m = __func_start_regexp__.match(s)
     if m:
         __infunc__ = m.group("func") or "__anonymous"
-        ast.handleMethodFlags(statements, __infunc__, m, d)
+        ast.handleMethodFlags(statements, __infunc__, m)
         return
 
     m = __def_regexp__.match(s)
@@ -212,26 +217,26 @@ def feeder(lineno, s, fn, root, d, statements):
 
     m = __export_func_regexp__.match(s)
     if m:
-        ast.handleExportFuncs(statements, m, classes, d)
+        ast.handleExportFuncs(statements, m, classes)
         return
 
     m = __addtask_regexp__.match(s)
     if m:
-        ast.handleAddTask(statements, m, d)
+        ast.handleAddTask(statements, m)
         return
 
     m = __addhandler_regexp__.match(s)
     if m:
-        ast.handleBBHandlers(statements, m, d)
+        ast.handleBBHandlers(statements, m)
         return
 
     m = __inherit_regexp__.match(s)
     if m:
-        ast.handleInherit(statements, m, d)
+        ast.handleInherit(statements, m)
         return
 
     from bb.parse import ConfHandler
-    return ConfHandler.feeder(lineno, s, fn, d, statements)
+    return ConfHandler.feeder(lineno, s, fn, statements)
 
 # Add us to the handlers list
 from bb.parse import handlers
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index 69f2eea1b..839a66202 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -49,7 +49,7 @@ def init(data):
 def supports(fn, d):
     return fn[-5:] == ".conf"
 
-def include(statements, oldfn, fn, data, error_out):
+def include(oldfn, fn, data, error_out):
     """
 
     error_out If True a ParseError will be reaised if the to be included
@@ -70,13 +70,13 @@ def include(statements, oldfn, fn, data, error_out):
 
     from bb.parse import handle
     try:
-        ret = handle(fn, data, True, statements)
+        ret = handle(fn, data, True)
     except IOError:
         if error_out:
             raise ParseError("Could not %(error_out)s file %(fn)s" % vars() )
         bb.msg.debug(2, bb.msg.domain.Parsing, "CONF file '%s' not found" % fn)
 
-def handle(fn, data, include, statements):
+def handle(fn, data, include):
     init(data)
 
     if include == 0:
@@ -89,8 +89,8 @@ def handle(fn, data, include, statements):
     if include:
         bb.parse.mark_dependency(data, abs_fn)
 
+    statements = ast.StatementGroup()
     lineno = 0
-    bb.data.setVar('FILE', fn, data)
     while 1:
         lineno = lineno + 1
         s = f.readline()
@@ -103,32 +103,36 @@ def handle(fn, data, include, statements):
             s2 = f.readline()[:-1].strip()
             lineno = lineno + 1
             s = s[:-1] + s2
-        feeder(lineno, s, fn, data, statements)
+        feeder(lineno, s, fn, statements)
 
+    # DONE WITH PARSING... time to evaluate
+    bb.data.setVar('FILE', fn, data)
+    statements.eval(data)
     if oldfile:
         bb.data.setVar('FILE', oldfile, data)
+
     return data
 
-def feeder(lineno, s, fn, data, statements):
+def feeder(lineno, s, fn, statements):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
-        ast.handleData(statements, groupd, data)
+        ast.handleData(statements, groupd)
         return
 
     m = __include_regexp__.match(s)
     if m:
-        ast.handleInclude(statements, m, fn, lineno, data, False)
+        ast.handleInclude(statements, m, fn, lineno, False)
         return
 
     m = __require_regexp__.match(s)
     if m:
-        ast.handleInclude(statements, m, fn, lineno, data, True)
+        ast.handleInclude(statements, m, fn, lineno, True)
         return
 
     m = __export_regexp__.match(s)
     if m:
-        ast.handleExport(statements, m, data)
+        ast.handleExport(statements, m)
         return
 
     raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));