From f00e5cf964d0ea48d517f0b8f9ca2aba646ea807 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 14 Jan 2010 17:36:31 +0000 Subject: bitbake-dev: Sync with upstream Signed-off-by: Richard Purdie --- bitbake-dev/lib/bb/command.py | 1 - bitbake-dev/lib/bb/cooker.py | 10 +-- bitbake-dev/lib/bb/fetch/__init__.py | 50 +++++++++--- bitbake-dev/lib/bb/fetch/git.py | 42 +++++----- bitbake-dev/lib/bb/providers.py | 2 +- bitbake-dev/lib/bb/server/__init__.py | 1 + bitbake-dev/lib/bb/server/none.py | 145 ++++++++++++++++++++++++++++++++++ bitbake-dev/lib/bb/server/xmlrpc.py | 145 ++++++++++++++++++++++++++++++++++ bitbake-dev/lib/bb/utils.py | 16 +++- bitbake-dev/lib/bb/xmlrpcserver.py | 145 ---------------------------------- 10 files changed, 370 insertions(+), 187 deletions(-) create mode 100644 bitbake-dev/lib/bb/server/__init__.py create mode 100644 bitbake-dev/lib/bb/server/none.py create mode 100644 bitbake-dev/lib/bb/server/xmlrpc.py delete mode 100644 bitbake-dev/lib/bb/xmlrpcserver.py (limited to 'bitbake-dev/lib') diff --git a/bitbake-dev/lib/bb/command.py b/bitbake-dev/lib/bb/command.py index e7c3770ff..1a1bf00b3 100644 --- a/bitbake-dev/lib/bb/command.py +++ b/bitbake-dev/lib/bb/command.py @@ -57,7 +57,6 @@ class Command: async_cmds[command] = (method) def runCommand(self, commandline): - bb.debug("Running command %s" % commandline) try: command = commandline.pop(0) if command in CommandsSync.__dict__: diff --git a/bitbake-dev/lib/bb/cooker.py b/bitbake-dev/lib/bb/cooker.py index b2b237b4c..1bf7d4bd1 100644 --- a/bitbake-dev/lib/bb/cooker.py +++ b/bitbake-dev/lib/bb/cooker.py @@ -25,7 +25,8 @@ import sys, os, getopt, glob, copy, os.path, re, time import bb from bb import utils, data, parse, event, cache, providers, taskdata, runqueue -from bb import xmlrpcserver, command +from bb import command +import bb.server.xmlrpc import itertools, sre_constants class MultipleMatches(Exception): @@ -62,14 +63,13 @@ class BBCooker: Manages one bitbake build run """ - def __init__(self, configuration): + def __init__(self, configuration, server): self.status = None self.cache = None self.bb_cache = None - self.server = bb.xmlrpcserver.BitBakeXMLRPCServer(self) - #self.server.register_function(self.showEnvironment) + self.server = server.BitBakeServer(self) self.configuration = configuration @@ -680,7 +680,7 @@ class BBCooker: retval = False if not retval: self.command.finishAsyncCommand() - bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures)) + bb.event.fire(bb.event.BuildCompleted(buildname, item, self.configuration.event_data, failures)) return False return 0.5 diff --git a/bitbake-dev/lib/bb/fetch/__init__.py b/bitbake-dev/lib/bb/fetch/__init__.py index b8a00107e..ab4658bc3 100644 --- a/bitbake-dev/lib/bb/fetch/__init__.py +++ b/bitbake-dev/lib/bb/fetch/__init__.py @@ -99,6 +99,11 @@ def fetcher_init(d): pd.delDomain("BB_URI_HEADREVS") else: bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy) + + for m in methods: + if hasattr(m, "init"): + m.init(d) + # Make sure our domains exist pd.addDomain("BB_URI_HEADREVS") pd.addDomain("BB_URI_LOCALCOUNT") @@ -147,14 +152,16 @@ def init(urls, d, setup = True): urldata_cache[fn] = urldata return urldata -def go(d): +def go(d, urls = None): """ Fetch all urls init must have previously been called """ - urldata = init([], d, True) + if not urls: + urls = d.getVar("SRC_URI", 1).split() + urldata = init(urls, d, True) - for u in urldata: + for u in urls: ud = urldata[u] m = ud.method if ud.localfile: @@ -465,6 +472,23 @@ class Fetch(object): srcrev_internal_helper = staticmethod(srcrev_internal_helper) + def localcount_internal_helper(ud, d): + """ + Return: + a) a locked localcount if specified + b) None otherwise + """ + + localcount= None + if 'name' in ud.parm: + pn = data.getVar("PN", d, 1) + localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1) + if not localcount: + localcount = data.getVar("LOCALCOUNT", d, 1) + return localcount + + localcount_internal_helper = staticmethod(localcount_internal_helper) + def try_mirror(d, tarfn): """ Try to use a mirrored version of the sources. We do this @@ -553,12 +577,7 @@ class Fetch(object): """ """ - has_sortable_valid = hasattr(self, "_sortable_revision_valid") - has_sortable = hasattr(self, "_sortable_revision") - - if has_sortable and not has_sortable_valid: - return self._sortable_revision(url, ud, d) - elif has_sortable and self._sortable_revision_valid(url, ud, d): + if hasattr(self, "_sortable_revision"): return self._sortable_revision(url, ud, d) pd = persist_data.PersistData(d) @@ -566,13 +585,24 @@ class Fetch(object): latest_rev = self._build_revision(url, ud, d) last_rev = pd.getValue("BB_URI_LOCALCOUNT", key + "_rev") - count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count") + uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False + count = None + if uselocalcount: + count = Fetch.localcount_internal_helper(ud, d) + if count is None: + count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count") if last_rev == latest_rev: return str(count + "+" + latest_rev) + buildindex_provided = hasattr(self, "_sortable_buildindex") + if buildindex_provided: + count = self._sortable_buildindex(url, ud, d, latest_rev) + if count is None: count = "0" + elif uselocalcount or buildindex_provided: + count = str(count) else: count = str(int(count) + 1) diff --git a/bitbake-dev/lib/bb/fetch/git.py b/bitbake-dev/lib/bb/fetch/git.py index 911c5e437..43053d6c4 100644 --- a/bitbake-dev/lib/bb/fetch/git.py +++ b/bitbake-dev/lib/bb/fetch/git.py @@ -28,6 +28,12 @@ from bb.fetch import runfetchcmd class Git(Fetch): """Class to fetch a module or modules from git repositories""" + def init(self, d): + # + # Only enable _sortable revision if the key is set + # + if bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True): + self._sortable_buildindex = self._sortable_buildindex_disabled def supports(self, url, ud, d): """ Check to see if a given url can be fetched with git. @@ -145,44 +151,32 @@ class Git(Fetch): def _build_revision(self, url, ud, d): return ud.tag - def _sortable_revision_valid(self, url, ud, d): - return bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True) or False - - def _sortable_revision(self, url, ud, d): + def _sortable_buildindex_disabled(self, url, ud, d, rev): """ - This is only called when _sortable_revision_valid called true - - We will have to get the updated revision. + Return a suitable buildindex for the revision specified. This is done by counting revisions + using "git rev-list" which may or may not work in different circumstances. """ gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) - key = "GIT_CACHED_REVISION-%s-%s" % (gitsrcname, ud.tag) - if bb.data.getVar(key, d): - return bb.data.getVar(key, d) - - - # Runtime warning on wrongly configured sources - if ud.tag == "1": - bb.msg.error(1, bb.msg.domain.Fetcher, "SRCREV is '1'. This indicates a configuration error of %s" % url) - return "0+1" - cwd = os.getcwd() # Check if we have the rev already if not os.path.exists(repodir): - print "no repo" self.go(None, ud, d) + if not os.path.exists(repodir): + bb.msg.error(bb.msg.domain.Fetcher, "GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value" % (url, repodir)) + return None + os.chdir(repodir) - if not self._contains_ref(ud.tag, d): + if not self._contains_ref(rev, d): self.go(None, ud, d) - output = runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % ud.tag, d, quiet=True) + output = runfetchcmd("git rev-list %s -- 2> /dev/null | wc -l" % rev, d, quiet=True) os.chdir(cwd) - sortable_revision = "%s+%s" % (output.split()[0], ud.tag) - bb.data.setVar(key, sortable_revision, d) - return sortable_revision - + buildindex = "%s" % output.split()[0] + bb.msg.debug(1, bb.msg.domain.Fetcher, "GIT repository for %s in %s is returning %s revisions in rev-list before %s" % (url, repodir, buildindex, rev)) + return buildindex diff --git a/bitbake-dev/lib/bb/providers.py b/bitbake-dev/lib/bb/providers.py index 8970fb3be..6c1cf78eb 100644 --- a/bitbake-dev/lib/bb/providers.py +++ b/bitbake-dev/lib/bb/providers.py @@ -191,7 +191,7 @@ def _filterProviders(providers, item, cfgData, dataCache): eligible.append(preferred_versions[pn][1]) # Now add latest verisons - for pn in pkg_pn.keys(): + for pn in sortpkg_pn.keys(): if pn in preferred_versions and preferred_versions[pn][1]: continue preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0]) diff --git a/bitbake-dev/lib/bb/server/__init__.py b/bitbake-dev/lib/bb/server/__init__.py new file mode 100644 index 000000000..0dd04cf72 --- /dev/null +++ b/bitbake-dev/lib/bb/server/__init__.py @@ -0,0 +1 @@ +import xmlrpc diff --git a/bitbake-dev/lib/bb/server/none.py b/bitbake-dev/lib/bb/server/none.py new file mode 100644 index 000000000..ef061bc5d --- /dev/null +++ b/bitbake-dev/lib/bb/server/none.py @@ -0,0 +1,145 @@ +# +# BitBake XMLRPC Server +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements an xmlrpc server for BitBake. + + Use this by deriving a class from BitBakeXMLRPCServer and then adding + methods which you want to "export" via XMLRPC. If the methods have the + prefix xmlrpc_, then registering those function will happen automatically, + if not, you need to call register_function. + + Use register_idle_function() to add a function which the xmlrpc server + calls from within server_forever when no requests are pending. Make sure + that those functions are non-blocking or else you will introduce latency + in the server's main loop. +""" + +import bb +import xmlrpclib + +DEBUG = False + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import inspect, select + +class BitBakeServerCommands(): + def __init__(self, server, cooker): + self.cooker = cooker + self.server = server + + def registerEventHandler(self, host, port): + """ + Register a remote UI Event Handler + """ + s = xmlrpclib.Server("http://%s:%d" % (host, port), allow_none=True) + return bb.event.register_UIHhandler(s) + + def unregisterEventHandler(self, handlerNum): + """ + Unregister a remote UI Event Handler + """ + return bb.event.unregister_UIHhandler(handlerNum) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + return self.cooker.command.runCommand(command) + + def terminateServer(self): + """ + Trigger the server to quit + """ + self.server.quit = True + print "Server (cooker) exitting" + return + + def ping(self): + """ + Dummy method which can be used to check the server is still alive + """ + return True + +class BitBakeXMLRPCServer(SimpleXMLRPCServer): + # remove this when you're done with debugging + # allow_reuse_address = True + + def __init__(self, cooker, interface = ("localhost", 0)): + """ + Constructor + """ + SimpleXMLRPCServer.__init__(self, interface, + requestHandler=SimpleXMLRPCRequestHandler, + logRequests=False, allow_none=True) + self._idlefuns = {} + self.host, self.port = self.socket.getsockname() + #self.register_introspection_functions() + commands = BitBakeServerCommands(self, cooker) + self.autoregister_all_functions(commands, "") + + def autoregister_all_functions(self, context, prefix): + """ + Convenience method for registering all functions in the scope + of this class that start with a common prefix + """ + methodlist = inspect.getmembers(context, inspect.ismethod) + for name, method in methodlist: + if name.startswith(prefix): + self.register_function(method, name[len(prefix):]) + + def register_idle_function(self, function, data): + """Register a function to be called while the server is idle""" + assert callable(function) + self._idlefuns[function] = data + + def serve_forever(self): + """ + Serve Requests. Overloaded to honor a quit command + """ + self.quit = False + while not self.quit: + #print "Idle queue length %s" % len(self._idlefuns) + if len(self._idlefuns) == 0: + self.timeout = None + else: + self.timeout = 0 + self.handle_request() + #print "Idle timeout, running idle functions" + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, False) + if not retval: + del self._idlefuns[function] + except SystemExit: + raise + except: + import traceback + traceback.print_exc() + pass + + # Tell idle functions we're exiting + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, True) + except: + pass + + self.server_close() + return diff --git a/bitbake-dev/lib/bb/server/xmlrpc.py b/bitbake-dev/lib/bb/server/xmlrpc.py new file mode 100644 index 000000000..c5937abd6 --- /dev/null +++ b/bitbake-dev/lib/bb/server/xmlrpc.py @@ -0,0 +1,145 @@ +# +# BitBake XMLRPC Server +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements an xmlrpc server for BitBake. + + Use this by deriving a class from BitBakeXMLRPCServer and then adding + methods which you want to "export" via XMLRPC. If the methods have the + prefix xmlrpc_, then registering those function will happen automatically, + if not, you need to call register_function. + + Use register_idle_function() to add a function which the xmlrpc server + calls from within server_forever when no requests are pending. Make sure + that those functions are non-blocking or else you will introduce latency + in the server's main loop. +""" + +import bb +import xmlrpclib + +DEBUG = False + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import inspect, select + +class BitBakeServerCommands(): + def __init__(self, server, cooker): + self.cooker = cooker + self.server = server + + def registerEventHandler(self, host, port): + """ + Register a remote UI Event Handler + """ + s = xmlrpclib.Server("http://%s:%d" % (host, port), allow_none=True) + return bb.event.register_UIHhandler(s) + + def unregisterEventHandler(self, handlerNum): + """ + Unregister a remote UI Event Handler + """ + return bb.event.unregister_UIHhandler(handlerNum) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + return self.cooker.command.runCommand(command) + + def terminateServer(self): + """ + Trigger the server to quit + """ + self.server.quit = True + print "Server (cooker) exitting" + return + + def ping(self): + """ + Dummy method which can be used to check the server is still alive + """ + return True + +class BitBakeServer(SimpleXMLRPCServer): + # remove this when you're done with debugging + # allow_reuse_address = True + + def __init__(self, cooker, interface = ("localhost", 0)): + """ + Constructor + """ + SimpleXMLRPCServer.__init__(self, interface, + requestHandler=SimpleXMLRPCRequestHandler, + logRequests=False, allow_none=True) + self._idlefuns = {} + self.host, self.port = self.socket.getsockname() + #self.register_introspection_functions() + commands = BitBakeServerCommands(self, cooker) + self.autoregister_all_functions(commands, "") + + def autoregister_all_functions(self, context, prefix): + """ + Convenience method for registering all functions in the scope + of this class that start with a common prefix + """ + methodlist = inspect.getmembers(context, inspect.ismethod) + for name, method in methodlist: + if name.startswith(prefix): + self.register_function(method, name[len(prefix):]) + + def register_idle_function(self, function, data): + """Register a function to be called while the server is idle""" + assert callable(function) + self._idlefuns[function] = data + + def serve_forever(self): + """ + Serve Requests. Overloaded to honor a quit command + """ + self.quit = False + while not self.quit: + #print "Idle queue length %s" % len(self._idlefuns) + if len(self._idlefuns) == 0: + self.timeout = None + else: + self.timeout = 0 + self.handle_request() + #print "Idle timeout, running idle functions" + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, False) + if not retval: + del self._idlefuns[function] + except SystemExit: + raise + except: + import traceback + traceback.print_exc() + pass + + # Tell idle functions we're exiting + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, True) + except: + pass + + self.server_close() + return diff --git a/bitbake-dev/lib/bb/utils.py b/bitbake-dev/lib/bb/utils.py index 603c92642..5b0aaba4a 100644 --- a/bitbake-dev/lib/bb/utils.py +++ b/bitbake-dev/lib/bb/utils.py @@ -21,8 +21,9 @@ BitBake Utility Functions digits = "0123456789" ascii_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +separators = ".-" -import re, fcntl, os +import re, fcntl, os, types def explode_version(s): r = [] @@ -39,12 +40,15 @@ def explode_version(s): r.append(m.group(1)) s = m.group(2) continue + r.append(s[0]) s = s[1:] return r def vercmp_part(a, b): va = explode_version(a) vb = explode_version(b) + sa = False + sb = False while True: if va == []: ca = None @@ -56,6 +60,16 @@ def vercmp_part(a, b): cb = vb.pop(0) if ca == None and cb == None: return 0 + + if type(ca) is types.StringType: + sa = ca in separators + if type(cb) is types.StringType: + sb = cb in separators + if sa and not sb: + return -1 + if not sa and sb: + return 1 + if ca > cb: return 1 if ca < cb: diff --git a/bitbake-dev/lib/bb/xmlrpcserver.py b/bitbake-dev/lib/bb/xmlrpcserver.py deleted file mode 100644 index ef061bc5d..000000000 --- a/bitbake-dev/lib/bb/xmlrpcserver.py +++ /dev/null @@ -1,145 +0,0 @@ -# -# BitBake XMLRPC Server -# -# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer -# Copyright (C) 2006 - 2008 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -""" - This module implements an xmlrpc server for BitBake. - - Use this by deriving a class from BitBakeXMLRPCServer and then adding - methods which you want to "export" via XMLRPC. If the methods have the - prefix xmlrpc_, then registering those function will happen automatically, - if not, you need to call register_function. - - Use register_idle_function() to add a function which the xmlrpc server - calls from within server_forever when no requests are pending. Make sure - that those functions are non-blocking or else you will introduce latency - in the server's main loop. -""" - -import bb -import xmlrpclib - -DEBUG = False - -from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler -import inspect, select - -class BitBakeServerCommands(): - def __init__(self, server, cooker): - self.cooker = cooker - self.server = server - - def registerEventHandler(self, host, port): - """ - Register a remote UI Event Handler - """ - s = xmlrpclib.Server("http://%s:%d" % (host, port), allow_none=True) - return bb.event.register_UIHhandler(s) - - def unregisterEventHandler(self, handlerNum): - """ - Unregister a remote UI Event Handler - """ - return bb.event.unregister_UIHhandler(handlerNum) - - def runCommand(self, command): - """ - Run a cooker command on the server - """ - return self.cooker.command.runCommand(command) - - def terminateServer(self): - """ - Trigger the server to quit - """ - self.server.quit = True - print "Server (cooker) exitting" - return - - def ping(self): - """ - Dummy method which can be used to check the server is still alive - """ - return True - -class BitBakeXMLRPCServer(SimpleXMLRPCServer): - # remove this when you're done with debugging - # allow_reuse_address = True - - def __init__(self, cooker, interface = ("localhost", 0)): - """ - Constructor - """ - SimpleXMLRPCServer.__init__(self, interface, - requestHandler=SimpleXMLRPCRequestHandler, - logRequests=False, allow_none=True) - self._idlefuns = {} - self.host, self.port = self.socket.getsockname() - #self.register_introspection_functions() - commands = BitBakeServerCommands(self, cooker) - self.autoregister_all_functions(commands, "") - - def autoregister_all_functions(self, context, prefix): - """ - Convenience method for registering all functions in the scope - of this class that start with a common prefix - """ - methodlist = inspect.getmembers(context, inspect.ismethod) - for name, method in methodlist: - if name.startswith(prefix): - self.register_function(method, name[len(prefix):]) - - def register_idle_function(self, function, data): - """Register a function to be called while the server is idle""" - assert callable(function) - self._idlefuns[function] = data - - def serve_forever(self): - """ - Serve Requests. Overloaded to honor a quit command - """ - self.quit = False - while not self.quit: - #print "Idle queue length %s" % len(self._idlefuns) - if len(self._idlefuns) == 0: - self.timeout = None - else: - self.timeout = 0 - self.handle_request() - #print "Idle timeout, running idle functions" - for function, data in self._idlefuns.items(): - try: - retval = function(self, data, False) - if not retval: - del self._idlefuns[function] - except SystemExit: - raise - except: - import traceback - traceback.print_exc() - pass - - # Tell idle functions we're exiting - for function, data in self._idlefuns.items(): - try: - retval = function(self, data, True) - except: - pass - - self.server_close() - return -- cgit v1.2.3