bitbake: Implement signatures

Includes functionality to find out what changes between two different singature data dumps.

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

6 files changed, 56 insertions, 5 deletions

diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 21cbad915..646fdecbe 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -38,7 +38,7 @@ except ImportError:
     import pickle
     bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
 
-__cache_version__ = "131"
+__cache_version__ = "132"
 
 class Cache:
     """
@@ -377,6 +377,10 @@ class Cache:
 
         cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True)
 
+        cacheData.tasks[file_name] = self.getVar('__BBTASKS', file_name, True)
+        for t in cacheData.tasks[file_name]:
+            cacheData.basetaskhash[file_name + "." + t] = self.getVar("BB_BASEHASH_task-%s" % t, file_name, True)
+
         # build FileName to PackageName lookup table
         cacheData.pkg_fn[file_name] = pn
         cacheData.pkg_pepvpr[file_name] = (pe, pv, pr)
@@ -539,6 +543,8 @@ class CacheData:
         self.task_deps = {}
         self.stamp = {}
         self.preferred = {}
+        self.tasks = {}
+        self.basetaskhash = {}
 
         """
         Indirect Cache variables
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ea33693dd..5cea9dba9 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -496,6 +496,8 @@ class BBCooker:
     def parseConfigurationFiles(self, files):
         try:
             data = self.configuration.data
+
+            bb.parse.init_parser(data)
             for f in files:
                 data = bb.parse.handle(f, data)
 
@@ -548,6 +550,8 @@ class BBCooker:
                 bb.fetch.fetcher_init(self.configuration.data)
             bb.codeparser.parser_cache_init(self.configuration.data)
 
+            bb.parse.init_parser(data)
+
             bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
 
         except IOError as e:
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index 95f372b00..c7249ef05 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -28,6 +28,7 @@ handlers = []
 
 import bb, os
 import bb.utils
+import bb.siggen
 
 class ParseError(Exception):
     """Exception raised when parsing fails"""
@@ -79,6 +80,9 @@ def init(fn, data):
         if h['supports'](fn):
             return h['init'](data)
 
+def init_parser(d):
+    bb.parse.siggen = bb.siggen.init(d)
+
 def resolve_file(fn, d):
     if not os.path.isabs(fn):
         bbpath = bb.data.getVar("BBPATH", d, True)
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index b80056943..870ae65b0 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -300,7 +300,7 @@ def handleInherit(statements, m):
     n = __word__.findall(files)
     statements.append(InheritNode(m.group(1)))
 
-def finalize(fn, d):
+def finalize(fn, d, variant = None):
     for lazykey in bb.data.getVar("__lazy_assigned", d) or ():
         if bb.data.getVar(lazykey, d) is None:
             val = bb.data.getVarFlag(lazykey, "defaultval", d)
@@ -323,7 +323,7 @@ def finalize(fn, d):
     tasklist = bb.data.getVar('__BBTASKS', d) or []
     bb.build.add_tasks(tasklist, d)
 
-    #bb.data.generate_dependencies(d)
+    bb.parse.siggen.finalise(fn, d, variant)
 
     bb.event.fire(bb.event.RecipeParsed(fn), d)
 
@@ -433,7 +433,7 @@ def multi_finalize(fn, d):
     for variant, variant_d in datastores.iteritems():
         if variant:
             try:
-                finalize(fn, variant_d)
+                finalize(fn, variant_d, variant)
             except bb.parse.SkipPackage:
                 bb.data.setVar("__SKIPPED", True, variant_d)
 
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 1b974312b..3d84bb17e 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -178,6 +178,7 @@ class RunQueueData:
         self.runq_task = []
         self.runq_depends = []
         self.runq_revdeps = []
+        self.runq_hash = []
 
     def runq_depends_names(self, ids):
         import re
@@ -477,6 +478,7 @@ class RunQueueData:
             self.runq_task.append(taskData.tasks_name[task])
             self.runq_depends.append(set(depends))
             self.runq_revdeps.append(set())
+            self.runq_hash.append("")
 
             runq_build.append(0)
             runq_recrdepends.append(recrdepends)
@@ -589,6 +591,7 @@ class RunQueueData:
                 del self.runq_depends[listid-delcount]
                 del runq_build[listid-delcount]
                 del self.runq_revdeps[listid-delcount]
+                del self.runq_hash[listid-delcount]
                 delcount = delcount + 1
                 maps.append(-1)
 
@@ -686,6 +689,20 @@ class RunQueueData:
             #bb.note("Found setscene for %s %s" % (self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task]))
             self.runq_setscene.append(task)
 
+        # Interate over the task list and call into the siggen code
+
+        dealtwith = set()
+        todeal = set(range(len(self.runq_fnid)))
+        while len(todeal) > 0:
+            for task in todeal.copy():
+                if len(self.runq_depends[task] - dealtwith) == 0:
+                    dealtwith.add(task)
+                    todeal.remove(task)
+                    procdep = []
+                    for dep in self.runq_depends[task]:
+                        procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
+                    self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
+
         return len(self.runq_fnid)
 
     def dump_data(self, taskQueue):
@@ -885,7 +902,10 @@ class RunQueue:
                 self.state = runQueueSceneInit
 
         if self.state is runQueueSceneInit:
-            self.rqexe = RunQueueExecuteScenequeue(self)
+            if self.cooker.configuration.dump_signatures:
+                self.dump_signatures()
+            else:
+                self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state is runQueueSceneRun:
             self.rqexe.execute()
@@ -926,6 +946,20 @@ class RunQueue:
         else:
             self.rqexe.finish()
 
+    def dump_signatures(self):
+        self.state = runQueueComplete
+        done = set()
+        bb.note("Reparsing files to collect dependency data")
+        for task in range(len(self.rqdata.runq_fnid)):
+            if self.rqdata.runq_fnid[task] not in done:
+                fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 
+                the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
+                done.add(self.rqdata.runq_fnid[task])
+
+        bb.parse.siggen.dump_sigs(self.rqdata.dataCache)
+
+        return
+
 
 class RunQueueExecute:
 
@@ -1007,6 +1041,8 @@ class RunQueueExecute:
                     comps = var.split("=")
                     env[comps[0]] = comps[1]
 
+            env['BB_TASKHASH'] = self.rqdata.runq_hash[task]
+
             sys.stdout.flush()
             sys.stderr.flush()
 
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 9fc56eaa0..c2e6ff08e 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -452,6 +452,7 @@ def preserved_envvars_list():
         'BB_PRESERVE_ENV',
         'BB_ENV_WHITELIST',
         'BB_ENV_EXTRAWHITE',
+        'BB_TASKHASH',
         'COLORTERM',
         'DBUS_SESSION_BUS_ADDRESS',
         'DESKTOP_SESSION',