diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2')
| -rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 1074 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/bzr.py | 141 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/cvs.py | 181 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 242 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/hg.py | 174 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/local.py | 80 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/osc.py | 135 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/perforce.py | 196 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/repo.py | 98 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/ssh.py | 120 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/svk.py | 97 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/svn.py | 180 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch2/wget.py | 91 |
13 files changed, 0 insertions, 2809 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py deleted file mode 100644 index 4e03fc988..000000000 --- a/bitbake/lib/bb/fetch2/__init__.py +++ /dev/null @@ -1,1074 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -from __future__ import absolute_import -from __future__ import print_function -import os, re -import logging -import bb -from bb import data -from bb import persist_data -from bb import utils - -__version__ = "2" - -logger = logging.getLogger("BitBake.Fetcher") - -class BBFetchException(Exception): - """Class all fetch exceptions inherit from""" - def __init__(self, message): - self.msg = message - Exception.__init__(self, message) - - def __str__(self): - return self.msg - -class MalformedUrl(BBFetchException): - """Exception raised when encountering an invalid url""" - def __init__(self, url): - msg = "The URL: '%s' is invalid and cannot be interpreted" % url - self.url = url - BBFetchException.__init__(self, msg) - self.args = url - -class FetchError(BBFetchException): - """General fetcher exception when something happens incorrectly""" - def __init__(self, message, url = None): - msg = "Fetcher failure for URL: '%s'. %s" % (url, message) - self.url = url - BBFetchException.__init__(self, msg) - self.args = (message, url) - -class UnpackError(BBFetchException): - """General fetcher exception when something happens incorrectly when unpacking""" - def __init__(self, message, url): - msg = "Unpack failure for URL: '%s'. %s" % (url, message) - self.url = url - BBFetchException.__init__(self, msg) - self.args = (message, url) - -class NoMethodError(BBFetchException): - """Exception raised when there is no method to obtain a supplied url or set of urls""" - def __init__(self, url): - msg = "Could not find a fetcher which supports the URL: '%s'" % url - self.url = url - BBFetchException.__init__(self, msg) - self.args = url - -class MissingParameterError(BBFetchException): - """Exception raised when a fetch method is missing a critical parameter in the url""" - def __init__(self, missing, url): - msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) - self.url = url - self.missing = missing - BBFetchException.__init__(self, msg) - self.args = (missing, url) - -class ParameterError(BBFetchException): - """Exception raised when a url cannot be proccessed due to invalid parameters.""" - def __init__(self, message, url): - msg = "URL: '%s' has invalid parameters. %s" % (url, message) - self.url = url - BBFetchException.__init__(self, msg) - self.args = (message, url) - -class MD5SumError(BBFetchException): - """Exception raised when a MD5 checksum of a file does not match for a downloaded file""" - def __init__(self, path, wanted, got, url): - msg = "File: '%s' has md5 checksum %s when %s was expected (from URL: '%s')" % (path, got, wanted, url) - self.url = url - self.path = path - self.wanted = wanted - self.got = got - BBFetchException.__init__(self, msg) - self.args = (path, wanted, got, url) - -class SHA256SumError(MD5SumError): - """Exception raised when a SHA256 checksum of a file does not match for a downloaded file""" - def __init__(self, path, wanted, got, url): - msg = "File: '%s' has sha256 checksum %s when %s was expected (from URL: '%s')" % (path, got, wanted, url) - self.url = url - self.path = path - self.wanted = wanted - self.got = got - BBFetchException.__init__(self, msg) - self.args = (path, wanted, got, url) - -class NetworkAccess(BBFetchException): - """Exception raised when network access is disabled but it is required.""" - def __init__(self, url, cmd): - msg = "Network access disabled through BB_NO_NETWORK but access rquested with command %s (for url %s)" % (cmd, url) - self.url = url - self.cmd = cmd - BBFetchException.__init__(self, msg) - self.args = (url, cmd) - - -def decodeurl(url): - """Decodes an URL into the tokens (scheme, network location, path, - user, password, parameters). - """ - - m = re.compile('(?P<type>[^:]*)://((?P<user>.+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) - if not m: - raise MalformedUrl(url) - - type = m.group('type') - location = m.group('location') - if not location: - raise MalformedUrl(url) - user = m.group('user') - parm = m.group('parm') - - locidx = location.find('/') - if locidx != -1 and type.lower() != 'file': - host = location[:locidx] - path = location[locidx:] - else: - host = "" - path = location - if user: - m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user) - if m: - user = m.group('user') - pswd = m.group('pswd') - else: - user = '' - pswd = '' - - p = {} - if parm: - for s in parm.split(';'): - s1, s2 = s.split('=') - p[s1] = s2 - - return (type, host, path, user, pswd, p) - -def encodeurl(decoded): - """Encodes a URL from tokens (scheme, network location, path, - user, password, parameters). - """ - - (type, host, path, user, pswd, p) = decoded - - if not path: - raise MissingParameterError('path', "encoded from the data %s" % str(decoded)) - if not type: - raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) - url = '%s://' % type - if user and type != "file": - url += "%s" % user - if pswd: - url += ":%s" % pswd - url += "@" - if host and type != "file": - url += "%s" % host - url += "%s" % path - if p: - for parm in p: - url += ";%s=%s" % (parm, p[parm]) - - return url - -def uri_replace(ud, uri_find, uri_replace, d): - if not ud.url or not uri_find or not uri_replace: - logger.debug(1, "uri_replace: passed an undefined value, not replacing") - uri_decoded = list(decodeurl(ud.url)) - uri_find_decoded = list(decodeurl(uri_find)) - uri_replace_decoded = list(decodeurl(uri_replace)) - result_decoded = ['', '', '', '', '', {}] - for i in uri_find_decoded: - loc = uri_find_decoded.index(i) - result_decoded[loc] = uri_decoded[loc] - if isinstance(i, basestring): - if (re.match(i, uri_decoded[loc])): - result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) - if uri_find_decoded.index(i) == 2: - if ud.mirrortarball: - result_decoded[loc] = os.path.join(os.path.dirname(result_decoded[loc]), os.path.basename(ud.mirrortarball)) - elif ud.localpath: - result_decoded[loc] = os.path.join(os.path.dirname(result_decoded[loc]), os.path.basename(ud.localpath)) - else: - return ud.url - return encodeurl(result_decoded) - -methods = [] -urldata_cache = {} -saved_headrevs = {} - -def fetcher_init(d): - """ - Called to initialize the fetchers once the configuration data is known. - Calls before this must not hit the cache. - """ - pd = persist_data.persist(d) - # When to drop SCM head revisions controlled by user policy - srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, True) or "clear" - if srcrev_policy == "cache": - logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) - elif srcrev_policy == "clear": - logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) - try: - bb.fetch2.saved_headrevs = pd['BB_URI_HEADREVS'].items() - except: - pass - del pd['BB_URI_HEADREVS'] - else: - raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) - - for m in methods: - if hasattr(m, "init"): - m.init(d) - -def fetcher_compare_revisions(d): - """ - Compare the revisions in the persistant cache with current values and - return true/false on whether they've changed. - """ - - pd = persist_data.persist(d) - data = pd['BB_URI_HEADREVS'].items() - data2 = bb.fetch2.saved_headrevs - - changed = False - for key in data: - if key not in data2 or data2[key] != data[key]: - logger.debug(1, "%s changed", key) - changed = True - return True - else: - logger.debug(2, "%s did not change", key) - return False - -def mirror_from_string(data): - return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ] - -def verify_checksum(u, ud, d): - """ - verify the MD5 and SHA256 checksum for downloaded src - - return value: - - True: checksum matched - - False: checksum unmatched - - if checksum is missing in recipes file, "BB_STRICT_CHECKSUM" decide the return value. - if BB_STRICT_CHECKSUM = "1" then return false as unmatched, otherwise return true as - matched - """ - - if not ud.type in ["http", "https", "ftp", "ftps"]: - return - - md5data = bb.utils.md5_file(ud.localpath) - sha256data = bb.utils.sha256_file(ud.localpath) - - if (ud.md5_expected == None or ud.sha256_expected == None): - logger.warn('Missing SRC_URI checksum for %s, consider adding to the recipe:\n' - 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"', - ud.localpath, ud.md5_name, md5data, - ud.sha256_name, sha256data) - if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1": - raise FetchError("No checksum specified for %s." % u, u) - return - - if ud.md5_expected != md5data: - raise MD5SumError(ud.localpath, ud.md5_expected, md5data, u) - - if ud.sha256_expected != sha256data: - raise SHA256SumError(ud.localpath, ud.sha256_expected, sha256data, u) - -def subprocess_setup(): - import signal - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - # SIGPIPE errors are known issues with gzip/bash - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - -def get_autorev(d): - # only not cache src rev in autorev case - if bb.data.getVar('BB_SRCREV_POLICY', d, True) != "cache": - bb.data.setVar('__BB_DONT_CACHE', '1', d) - return "AUTOINC" - -def get_srcrev(d): - """ - Return the version string for the current package - (usually to be used as PV) - Most packages usually only have one SCM so we just pass on the call. - In the multi SCM case, we build a value based on SRCREV_FORMAT which must - have been set. - """ - - scms = [] - fetcher = Fetch(bb.data.getVar('SRC_URI', d, True).split(), d) - urldata = fetcher.ud - for u in urldata: - if urldata[u].method.supports_srcrev(): - scms.append(u) - - if len(scms) == 0: - raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") - - if len(scms) == 1 and len(urldata[scms[0]].names) == 1: - return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0]) - - # - # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT - # - format = bb.data.getVar('SRCREV_FORMAT', d, True) - if not format: - raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") - - for scm in scms: - ud = urldata[scm] - for name in ud.names: - rev = ud.method.sortable_revision(scm, ud, d, name) - format = format.replace(name, rev) - - return format - -def localpath(url, d): - fetcher = bb.fetch2.Fetch([url], d) - return fetcher.localpath(url) - -def runfetchcmd(cmd, d, quiet = False, cleanup = []): - """ - Run cmd returning the command output - Raise an error if interrupted or cmd fails - Optionally echo command output to stdout - Optionally remove the files/directories listed in cleanup upon failure - """ - - # Need to export PATH as binary could be in metadata paths - # rather than host provided - # Also include some other variables. - # FIXME: Should really include all export varaiables? - exportvars = ['PATH', 'GIT_PROXY_COMMAND', 'GIT_PROXY_HOST', - 'GIT_PROXY_PORT', 'GIT_CONFIG', 'http_proxy', 'ftp_proxy', - 'https_proxy', 'no_proxy', 'ALL_PROXY', 'all_proxy', - 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 'HOME'] - - for var in exportvars: - val = data.getVar(var, d, True) - if val: - cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) - - logger.debug(1, "Running %s", cmd) - - # redirect stderr to stdout - stdout_handle = os.popen(cmd + " 2>&1", "r") - output = "" - - while True: - line = stdout_handle.readline() - if not line: - break - if not quiet: - print(line, end=' ') - output += line - - status = stdout_handle.close() or 0 - signal = status >> 8 - exitstatus = status & 0xff - - if (signal or status != 0): - for f in cleanup: - try: - bb.utils.remove(f, True) - except OSError: - pass - - if signal: - raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) - elif status != 0: - raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) - - return output - -def check_network_access(d, info = "", url = None): - """ - log remote network access, and error if BB_NO_NETWORK is set - """ - if bb.data.getVar("BB_NO_NETWORK", d, True) == "1": - raise NetworkAccess(url, info) - else: - logger.debug(1, "Fetcher accessed the network with the command %s" % info) - -def try_mirrors(d, origud, mirrors, check = False): - """ - Try to use a mirrored version of the sources. - This method will be automatically called before the fetchers go. - - d Is a bb.data instance - uri is the original uri we're trying to download - mirrors is the list of mirrors we're going to try - """ - ld = d.createCopy() - for line in mirrors: - try: - (find, replace) = line - except ValueError: - continue - newuri = uri_replace(origud, find, replace, ld) - if newuri == origud.url: - continue - try: - ud = FetchData(newuri, ld) - ud.setup_localpath(ld) - - if check: - found = ud.method.checkstatus(newuri, ud, ld) - if found: - return found - continue - - if ud.method.need_update(newuri, ud, ld): - ud.method.download(newuri, ud, ld) - if hasattr(ud.method,"build_mirror_data"): - ud.method.build_mirror_data(newuri, ud, ld) - - if not ud.localpath or not os.path.exists(ud.localpath): - continue - - if ud.localpath == origud.localpath: - return ud.localpath - - # We may be obtaining a mirror tarball which needs further processing by the real fetcher - # If that tarball is a local file:// we need to provide a symlink to it - dldir = ld.getVar("DL_DIR", True) - if os.path.basename(ud.localpath) != os.path.basename(origud.localpath): - dest = os.path.join(dldir, os.path.basename(ud.localpath)) - if not os.path.exists(dest): - os.symlink(ud.localpath, dest) - return None - # Otherwise the result is a local file:// and we symlink to it - if not os.path.exists(origud.localpath): - os.symlink(ud.localpath, origud.localpath) - return ud.localpath - - except bb.fetch2.NetworkAccess: - raise - - except bb.fetch2.BBFetchException as e: - logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) - logger.debug(1, str(e)) - try: - if os.path.isfile(ud.localpath): - bb.utils.remove(ud.localpath) - except UnboundLocalError: - pass - continue - return None - -def srcrev_internal_helper(ud, d, name): - """ - Return: - a) a source revision if specified - b) latest revision if SRCREV="AUTOINC" - c) None if not specified - """ - - if 'rev' in ud.parm: - return ud.parm['rev'] - - if 'tag' in ud.parm: - return ud.parm['tag'] - - rev = None - pn = data.getVar("PN", d, True) - if name != '': - rev = data.getVar("SRCREV_%s_pn-%s" % (name, pn), d, True) - if not rev: - rev = data.getVar("SRCREV_%s" % name, d, True) - if not rev: - rev = data.getVar("SRCREV_pn-%s" % pn, d, True) - if not rev: - rev = data.getVar("SRCREV", d, True) - if rev == "INVALID": - raise FetchError("Please set SRCREV to a valid value", ud.url) - if rev == "AUTOINC": - rev = ud.method.latest_revision(ud.url, ud, d, name) - - return rev - -class FetchData(object): - """ - A class which represents the fetcher state for a given URI. - """ - def __init__(self, url, d): - # localpath is the location of a downloaded result. If not set, the file is local. - self.donestamp = None - self.localfile = "" - self.localpath = None - self.lockfile = None - self.mirrortarball = None - (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d)) - self.date = self.getSRCDate(d) - self.url = url - if not self.user and "user" in self.parm: - self.user = self.parm["user"] - if not self.pswd and "pswd" in self.parm: - self.pswd = self.parm["pswd"] - self.setup = False - - if "name" in self.parm: - self.md5_name = "%s.md5sum" % self.parm["name"] - self.sha256_name = "%s.sha256sum" % self.parm["name"] - else: - self.md5_name = "md5sum" - self.sha256_name = "sha256sum" - self.md5_expected = bb.data.getVarFlag("SRC_URI", self.md5_name, d) - self.sha256_expected = bb.data.getVarFlag("SRC_URI", self.sha256_name, d) - - self.names = self.parm.get("name",'default').split(',') - - self.method = None - for m in methods: - if m.supports(url, self, d): - self.method = m - break - - if not self.method: - raise NoMethodError(url) - - if self.method.supports_srcrev(): - self.revisions = {} - for name in self.names: - self.revisions[name] = srcrev_internal_helper(self, d, name) - - # add compatibility code for non name specified case - if len(self.names) == 1: - self.revision = self.revisions[self.names[0]] - - if hasattr(self.method, "urldata_init"): - self.method.urldata_init(self, d) - - if "localpath" in self.parm: - # if user sets localpath for file, use it instead. - self.localpath = self.parm["localpath"] - self.basename = os.path.basename(self.localpath) - elif self.localfile: - self.localpath = self.method.localpath(self.url, self, d) - - if self.localfile and self.localpath: - # Note: These files should always be in DL_DIR whereas localpath may not be. - basepath = bb.data.expand("${DL_DIR}/%s" % os.path.basename(self.localpath), d) - self.donestamp = basepath + '.done' - self.lockfile = basepath + '.lock' - - def setup_localpath(self, d): - if not self.localpath: - self.localpath = self.method.localpath(self.url, self, d) - - def getSRCDate(self, d): - """ - Return the SRC Date for the component - - d the bb.data module - """ - if "srcdate" in self.parm: - return self.parm['srcdate'] - - pn = data.getVar("PN", d, True) - - if pn: - return data.getVar("SRCDATE_%s" % pn, d, True) or data.getVar("SRCDATE", d, True) or data.getVar("DATE", d, True) - - return data.getVar("SRCDATE", d, True) or data.getVar("DATE", d, True) - -class FetchMethod(object): - """Base class for 'fetch'ing data""" - - def __init__(self, urls = []): - self.urls = [] - - def supports(self, url, urldata, d): - """ - Check to see if this fetch class supports a given url. - """ - return 0 - - def localpath(self, url, urldata, d): - """ - Return the local filename of a given url assuming a successful fetch. - Can also setup variables in urldata for use in go (saving code duplication - and duplicate code execution) - """ - return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile) - - def _strip_leading_slashes(self, relpath): - """ - Remove leading slash as os.path.join can't cope - """ - while os.path.isabs(relpath): - relpath = relpath[1:] - return relpath - - def setUrls(self, urls): - self.__urls = urls - - def getUrls(self): - return self.__urls - - urls = property(getUrls, setUrls, None, "Urls property") - - def need_update(self, url, ud, d): - """ - Force a fetch, even if localpath exists? - """ - if os.path.exists(ud.localpath): - return False - return True - - def supports_srcrev(self): - """ - The fetcher supports auto source revisions (SRCREV) - """ - return False - - def download(self, url, urldata, d): - """ - Fetch urls - Assumes localpath was called first - """ - raise NoMethodError(url) - - def unpack(self, urldata, rootdir, data): - import subprocess - iterate = False - file = urldata.localpath - - try: - unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) - except ValueError, exc: - bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % - (file, urldata.parm.get('unpack'))) - - dots = file.split(".") - if dots[-1] in ['gz', 'bz2', 'Z']: - efile = os.path.join(bb.data.getVar('WORKDIR', data, True),os.path.basename('.'.join(dots[0:-1]))) - else: - efile = file - cmd = None - - if unpack: - if file.endswith('.tar'): - cmd = 'tar x --no-same-owner -f %s' % file - elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): - cmd = 'tar xz --no-same-owner -f %s' % file - elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): - cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file - elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): - cmd = 'gzip -dc %s > %s' % (file, efile) - elif file.endswith('.bz2'): - cmd = 'bzip2 -dc %s > %s' % (file, efile) - elif file.endswith('.tar.xz'): - cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file - elif file.endswith('.xz'): - cmd = 'xz -dc %s > %s' % (file, efile) - elif file.endswith('.zip') or file.endswith('.jar'): - try: - dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) - except ValueError, exc: - bb.fatal("Invalid value for 'dos' parameter for %s: %s" % - (file, urldata.parm.get('dos'))) - cmd = 'unzip -q -o' - if dos: - cmd = '%s -a' % cmd - cmd = "%s '%s'" % (cmd, file) - elif file.endswith('.src.rpm') or file.endswith('.srpm'): - if 'extract' in urldata.parm: - unpack_file = urldata.parm.get('extract') - cmd = 'rpm2cpio.sh %s | cpio -i %s' % (file, unpack_file) - iterate = True - iterate_file = unpack_file - else: - cmd = 'rpm2cpio.sh %s | cpio -i' % (file) - - if not unpack or not cmd: - # If file == dest, then avoid any copies, as we already put the file into dest! - dest = os.path.join(rootdir, os.path.basename(file)) - if (file != dest) and not (os.path.exists(dest) and os.path.samefile(file, dest)): - if os.path.isdir(file): - filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, True)) - destdir = "." - if file[0:len(filesdir)] == filesdir: - destdir = file[len(filesdir):file.rfind('/')] - destdir = destdir.strip('/') - if len(destdir) < 1: - destdir = "." - elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK): - os.makedirs("%s/%s" % (rootdir, destdir)) - cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir) - #cmd = 'tar -cf - -C "%d" -ps . | tar -xf - -C "%s/%s/"' % (file, rootdir, destdir) - else: - # The "destdir" handling was specifically done for FILESPATH - # items. So, only do so for file:// entries. - if urldata.type == "file" and urldata.path.find("/") != -1: - destdir = urldata.path.rsplit("/", 1)[0] - else: - destdir = "." - bb.mkdirhier("%s/%s" % (rootdir, destdir)) - cmd = 'cp %s %s/%s/' % (file, rootdir, destdir) - - if not cmd: - return - - # Change to subdir before executing command - save_cwd = os.getcwd(); - os.chdir(rootdir) - if 'subdir' in urldata.parm: - newdir = ("%s/%s" % (rootdir, urldata.parm.get('subdir'))) - bb.mkdirhier(newdir) - os.chdir(newdir) - - cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, True), cmd) - bb.note("Unpacking %s to %s/" % (file, os.getcwd())) - ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True) - - os.chdir(save_cwd) - - if ret != 0: - raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url) - - if iterate is True: - iterate_urldata = urldata - iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file) - self.unpack(urldata, rootdir, data) - - return - - def clean(self, urldata, d): - """ - Clean any existing full or partial download - """ - bb.utils.remove(urldata.localpath) - - def try_premirror(self, url, urldata, d): - """ - Should premirrors be used? - """ - return True - - def checkstatus(self, url, urldata, d): - """ - Check the status of a URL - Assumes localpath was called first - """ - logger.info("URL %s could not be checked for status since no method exists.", url) - return True - - def localcount_internal_helper(ud, d, name): - """ - Return: - a) a locked localcount if specified - b) None otherwise - """ - - localcount = None - if name != '': - pn = data.getVar("PN", d, True) - localcount = data.getVar("LOCALCOUNT_" + name, d, True) - if not localcount: - localcount = data.getVar("LOCALCOUNT", d, True) - return localcount - - localcount_internal_helper = staticmethod(localcount_internal_helper) - - def latest_revision(self, url, ud, d, name): - """ - Look in the cache for the latest revision, if not present ask the SCM. - """ - if not hasattr(self, "_latest_revision"): - raise ParameterError("The fetcher for this URL does not support _latest_revision", url) - - pd = persist_data.persist(d) - revs = pd['BB_URI_HEADREVS'] - key = self.generate_revision_key(url, ud, d, name) - rev = revs[key] - if rev != None: - return str(rev) - - revs[key] = rev = self._latest_revision(url, ud, d, name) - return rev - - def sortable_revision(self, url, ud, d, name): - """ - - """ - if hasattr(self, "_sortable_revision"): - return self._sortable_revision(url, ud, d) - - pd = persist_data.persist(d) - localcounts = pd['BB_URI_LOCALCOUNT'] - key = self.generate_revision_key(url, ud, d, name) - - latest_rev = self._build_revision(url, ud, d, name) - last_rev = localcounts[key + '_rev'] - uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False - count = None - if uselocalcount: - count = FetchMethod.localcount_internal_helper(ud, d, name) - if count is None: - count = localcounts[key + '_count'] or "0" - - if last_rev == latest_rev: - return str(count + "+" + latest_rev) - - buildindex_provided = hasattr(self, "_sortable_buildindex") - if buildindex_provided: - count = self._sortable_buildindex(url, ud, d, latest_rev) - - if count is None: - count = "0" - elif uselocalcount or buildindex_provided: - count = str(count) - else: - count = str(int(count) + 1) - - localcounts[key + '_rev'] = latest_rev - localcounts[key + '_count'] = count - - return str(count + "+" + latest_rev) - - def generate_revision_key(self, url, ud, d, name): - key = self._revision_key(url, ud, d, name) - return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "") - -class Fetch(object): - def __init__(self, urls, d, cache = True): - if len(urls) == 0: - urls = d.getVar("SRC_URI", True).split() - self.urls = urls - self.d = d - self.ud = {} - - fn = bb.data.getVar('FILE', d, True) - if cache and fn in urldata_cache: - self.ud = urldata_cache[fn] - - for url in urls: - if url not in self.ud: - self.ud[url] = FetchData(url, d) - - if cache: - urldata_cache[fn] = self.ud - - def localpath(self, url): - if url not in self.urls: - self.ud[url] = FetchData(url, self.d) - - self.ud[url].setup_localpath(self.d) - return bb.data.expand(self.ud[url].localpath, self.d) - - def localpaths(self): - """ - Return a list of the local filenames, assuming successful fetch - """ - local = [] - - for u in self.urls: - ud = self.ud[u] - ud.setup_localpath(self.d) - local.append(ud.localpath) - - return local - - def download(self, urls = []): - """ - Fetch all urls - """ - if len(urls) == 0: - urls = self.urls - - network = bb.data.getVar("BB_NO_NETWORK", self.d, True) - premirroronly = (bb.data.getVar("BB_FETCH_PREMIRRORONLY", self.d, True) == "1") - - for u in urls: - ud = self.ud[u] - ud.setup_localpath(self.d) - m = ud.method - localpath = "" - - if not ud.localfile: - continue - - lf = bb.utils.lockfile(ud.lockfile) - - try: - bb.data.setVar("BB_NO_NETWORK", network, self.d) - - if not m.need_update(u, ud, self.d): - localpath = ud.localpath - elif m.try_premirror(u, ud, self.d): - logger.debug(1, "Trying PREMIRRORS") - mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True)) - localpath = try_mirrors(self.d, ud, mirrors, False) - - if premirroronly: - bb.data.setVar("BB_NO_NETWORK", "1", self.d) - - if not localpath and m.need_update(u, ud, self.d): - try: - logger.debug(1, "Trying Upstream") - m.download(u, ud, self.d) - if hasattr(m, "build_mirror_data"): - m.build_mirror_data(u, ud, self.d) - localpath = ud.localpath - - except bb.fetch2.NetworkAccess: - raise - - except BBFetchException as e: - logger.debug(1, str(e)) - # Remove any incomplete fetch - if os.path.isfile(ud.localpath): - bb.utils.remove(ud.localpath) - logger.debug(1, "Trying MIRRORS") - mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True)) - localpath = try_mirrors (self.d, ud, mirrors) - - if not localpath or not os.path.exists(localpath): - raise FetchError("Unable to fetch URL %s from any source." % u, u) - - if os.path.exists(ud.donestamp): - # Touch the done stamp file to show active use of the download - try: - os.utime(ud.donestamp, None) - except: - # Errors aren't fatal here - pass - else: - # Only check the checksums if we've not seen this item before, then create the stamp - verify_checksum(u, ud, self.d) - open(ud.donestamp, 'w').close() - - finally: - bb.utils.unlockfile(lf) - - def checkstatus(self, urls = []): - """ - Check all urls exist upstream - """ - - if len(urls) == 0: - urls = self.urls - - for u in urls: - ud = self.ud[u] - ud.setup_localpath(self.d) - m = ud.method - logger.debug(1, "Testing URL %s", u) - # First try checking uri, u, from PREMIRRORS - mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True)) - ret = try_mirrors(self.d, ud, mirrors, True) - if not ret: - # Next try checking from the original uri, u - try: - ret = m.checkstatus(u, ud, self.d) - except: - # Finally, try checking uri, u, from MIRRORS - mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True)) - ret = try_mirrors (self.d, ud, mirrors, True) - - if not ret: - raise FetchError("URL %s doesn't work" % u, u) - - def unpack(self, root, urls = []): - """ - Check all urls exist upstream - """ - - if len(urls) == 0: - urls = self.urls - - for u in urls: - ud = self.ud[u] - ud.setup_localpath(self.d) - - if bb.data.expand(self.localpath, self.d) is None: - continue - - if ud.lockfile: - lf = bb.utils.lockfile(ud.lockfile) - - ud.method.unpack(ud, root, self.d) - - if ud.lockfile: - bb.utils.unlockfile(lf) - - def clean(self, urls = []): - """ - Clean files that the fetcher gets or places - """ - - if len(urls) == 0: - urls = self.urls - - for url in urls: - if url not in self.ud: - self.ud[url] = FetchData(url, d) - ud = self.ud[url] - ud.setup_localpath(self.d) - - if not ud.localfile or self.localpath is None: - continue - - if ud.lockfile: - lf = bb.utils.lockfile(ud.lockfile) - - ud.method.clean(ud, self.d) - if ud.donestamp: - bb.utils.remove(ud.donestamp) - - if ud.lockfile: - bb.utils.unlockfile(lf) - -from . import cvs -from . import git -from . import local -from . import svn -from . import wget -from . import svk -from . import ssh -from . import perforce -from . import bzr -from . import hg -from . import osc -from . import repo - -methods.append(local.Local()) -methods.append(wget.Wget()) -methods.append(svn.Svn()) -methods.append(git.Git()) -methods.append(cvs.Cvs()) -methods.append(svk.Svk()) -methods.append(ssh.SSH()) -methods.append(perforce.Perforce()) -methods.append(bzr.Bzr()) -methods.append(hg.Hg()) -methods.append(osc.Osc()) -methods.append(repo.Repo()) diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py deleted file mode 100644 index 454961eff..000000000 --- a/bitbake/lib/bb/fetch2/bzr.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -BitBake 'Fetch' implementation for bzr. - -""" - -# Copyright (C) 2007 Ross Burton -# Copyright (C) 2007 Richard Purdie -# -# Classes for obtaining upstream sources for the -# BitBake build tools. -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger - -class Bzr(FetchMethod): - def supports(self, url, ud, d): - return ud.type in ['bzr'] - - def urldata_init(self, ud, d): - """ - init bzr specific variable within url data - """ - # Create paths to bzr checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath) - - if not ud.revision: - ud.revision = self.latest_revision(ud.url, ud, d) - - ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) - - def _buildbzrcommand(self, ud, d, command): - """ - Build up an bzr commandline based on ud - command is "fetch", "update", "revno" - """ - - basecmd = data.expand('${FETCHCMD_bzr}', d) - - proto = ud.parm.get('proto', 'http') - - bzrroot = ud.host + ud.path - - options = [] - - if command is "revno": - bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) - else: - if ud.revision: - options.append("-r %s" % ud.revision) - - if command is "fetch": - bzrcmd = "%s co %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) - elif command is "update": - bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid bzr command %s" % command, ud.url) - - return bzrcmd - - def download(self, loc, ud, d): - """Fetch url""" - - if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): - bzrcmd = self._buildbzrcommand(ud, d, "update") - logger.debug(1, "BZR Update %s", loc) - bb.fetch2.check_network_access(d, bzrcmd, ud.url) - os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) - runfetchcmd(bzrcmd, d) - else: - bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) - bzrcmd = self._buildbzrcommand(ud, d, "fetch") - bb.fetch2.check_network_access(d, bzrcmd, ud.url) - logger.debug(1, "BZR Checkout %s", loc) - bb.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", bzrcmd) - runfetchcmd(bzrcmd, d) - - os.chdir(ud.pkgdir) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" - - # tar them up to a defined filename - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath]) - - def supports_srcrev(self): - return True - - def _revision_key(self, url, ud, d, name): - """ - Return a unique key for the url - """ - return "bzr:" + ud.pkgdir - - def _latest_revision(self, url, ud, d, name): - """ - Return the latest upstream revision number - """ - logger.debug(2, "BZR fetcher hitting network for %s", url) - - bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url) - - output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True) - - return output.strip() - - def _sortable_revision(self, url, ud, d): - """ - Return a sortable revision number which in our case is the revision number - """ - - return self._build_revision(url, ud, d) - - def _build_revision(self, url, ud, d): - return ud.revision diff --git a/bitbake/lib/bb/fetch2/cvs.py b/bitbake/lib/bb/fetch2/cvs.py deleted file mode 100644 index 12d11e0d5..000000000 --- a/bitbake/lib/bb/fetch2/cvs.py +++ /dev/null @@ -1,181 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -#Based on functions from the base bb module, Copyright 2003 Holger Schurig -# - -import os -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger -from bb.fetch2 import runfetchcmd - -class Cvs(FetchMethod): - """ - Class to fetch a module or modules from cvs repositories - """ - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with cvs. - """ - return ud.type in ['cvs'] - - def urldata_init(self, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("module", ud.url) - ud.module = ud.parm["module"] - - ud.tag = ud.parm.get('tag', "") - - # Override the default date in certain cases - if 'date' in ud.parm: - ud.date = ud.parm['date'] - elif ud.tag: - ud.date = "" - - norecurse = '' - if 'norecurse' in ud.parm: - norecurse = '_norecurse' - - fullpath = '' - if 'fullpath' in ud.parm: - fullpath = '_fullpath' - - ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d) - - def need_update(self, url, ud, d): - if (ud.date == "now"): - return True - if not os.path.exists(ud.localpath): - return True - return False - - def download(self, loc, ud, d): - - method = ud.parm.get('method', 'pserver') - localdir = ud.parm.get('localdir', ud.module) - cvs_port = ud.parm.get('port', '') - - cvs_rsh = None - if method == "ext": - if "rsh" in ud.parm: - cvs_rsh = ud.parm["rsh"] - - if method == "dir": - cvsroot = ud.path - else: - cvsroot = ":" + method - cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True) - if cvsproxyhost: - cvsroot += ";proxy=" + cvsproxyhost - cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True) - if cvsproxyport: - cvsroot += ";proxyport=" + cvsproxyport - cvsroot += ":" + ud.user - if ud.pswd: - cvsroot += ":" + ud.pswd - cvsroot += "@" + ud.host + ":" + cvs_port + ud.path - - options = [] - if 'norecurse' in ud.parm: - options.append("-l") - if ud.date: - # treat YYYYMMDDHHMM specially for CVS - if len(ud.date) == 12: - options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) - else: - options.append("-D \"%s UTC\"" % ud.date) - if ud.tag: - options.append("-r %s" % ud.tag) - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - data.setVar('CVSROOT', cvsroot, localdata) - data.setVar('CVSCOOPTS', " ".join(options), localdata) - data.setVar('CVSMODULE', ud.module, localdata) - cvscmd = data.getVar('FETCHCOMMAND', localdata, True) - cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, True) - - if cvs_rsh: - cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) - cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) - - # create module directory - logger.debug(2, "Fetch: checking for module directory") - pkg = data.expand('${PN}', d) - pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) - moddir = os.path.join(pkgdir, localdir) - if os.access(os.path.join(moddir, 'CVS'), os.R_OK): - logger.info("Update " + loc) - bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url) - # update sources there - os.chdir(moddir) - cmd = cvsupdatecmd - else: - logger.info("Fetch " + loc) - # check out sources there - bb.mkdirhier(pkgdir) - os.chdir(pkgdir) - logger.debug(1, "Running %s", cvscmd) - bb.fetch2.check_network_access(d, cvscmd, ud.url) - cmd = cvscmd - - runfetchcmd(cmd, d, cleanup = [moddir]) - - if not os.access(moddir, os.R_OK): - raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude 'CVS'" - - # tar them up to a defined filename - if 'fullpath' in ud.parm: - os.chdir(pkgdir) - cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir) - else: - os.chdir(moddir) - os.chdir('..') - cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)) - - runfetchcmd(cmd, d, cleanup = [ud.localpath]) - - def clean(self, ud, d): - """ Clean CVS Files and tarballs """ - - pkg = data.expand('${PN}', d) - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) - - bb.utils.remove(pkgdir, True) - bb.utils.remove(ud.localpath) - diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py deleted file mode 100644 index f2c27e42a..000000000 --- a/bitbake/lib/bb/fetch2/git.py +++ /dev/null @@ -1,242 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' git implementation - -""" - -#Copyright (C) 2005 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger - -class Git(FetchMethod): - """Class to fetch a module or modules from git repositories""" - def init(self, d): - # - # Only enable _sortable revision if the key is set - # - if bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True): - self._sortable_buildindex = self._sortable_buildindex_disabled - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with git. - """ - return ud.type in ['git'] - - def urldata_init(self, ud, d): - """ - init git specific variable within url data - so that the git method like latest_revision() can work - """ - if 'protocol' in ud.parm: - ud.proto = ud.parm['protocol'] - elif not ud.host: - ud.proto = 'file' - else: - ud.proto = "rsync" - - ud.nocheckout = False - if 'nocheckout' in ud.parm: - ud.nocheckout = True - - branches = ud.parm.get("branch", "master").split(',') - if len(branches) != len(ud.names): - raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) - ud.branches = {} - for name in ud.names: - branch = branches[ud.names.index(name)] - ud.branches[name] = branch - - gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) - ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) - ud.fullmirror = os.path.join(data.getVar("DL_DIR", d, True), ud.mirrortarball) - ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) - - ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - - for name in ud.names: - # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one - if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): - ud.revisions[name] = self.latest_revision(ud.url, ud, d, name) - - ud.write_tarballs = (data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0" - - ud.localfile = ud.clonedir - - def localpath(self, url, ud, d): - return ud.clonedir - - def need_update(self, u, ud, d): - if not os.path.exists(ud.clonedir): - return True - os.chdir(ud.clonedir) - for name in ud.names: - if not self._contains_ref(ud.revisions[name], d): - return True - if ud.write_tarballs and not os.path.exists(ud.fullmirror): - return True - return False - - def try_premirror(self, u, ud, d): - # If we don't do this, updating an existing checkout with only premirrors - # is not possible - if bb.data.getVar("BB_FETCH_PREMIRRORONLY", d, True) is not None: - return True - if os.path.exists(ud.clonedir): - return False - return True - - def download(self, loc, ud, d): - """Fetch url""" - - if ud.user: - username = ud.user + '@' - else: - username = "" - - ud.repochanged = not os.path.exists(ud.fullmirror) - - # If the checkout doesn't exist and the mirror tarball does, extract it - if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): - bb.mkdirhier(ud.clonedir) - os.chdir(ud.clonedir) - runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) - - # If the repo still doesn't exist, fallback to cloning it - if not os.path.exists(ud.clonedir): - bb.fetch2.check_network_access(d, "git clone --bare %s%s" % (ud.host, ud.path)) - runfetchcmd("%s clone --bare %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) - - os.chdir(ud.clonedir) - # Update the checkout if needed - needupdate = False - for name in ud.names: - if not self._contains_ref(ud.revisions[name], d): - needupdate = True - if needupdate: - bb.fetch2.check_network_access(d, "git fetch %s%s" % (ud.host, ud.path), ud.url) - try: - runfetchcmd("%s remote prune origin" % ud.basecmd, d) - runfetchcmd("%s remote rm origin" % ud.basecmd, d) - except bb.fetch2.FetchError: - logger.debug(1, "No Origin") - - runfetchcmd("%s remote add origin %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) - runfetchcmd("%s fetch --all -t" % ud.basecmd, d) - runfetchcmd("%s prune-packed" % ud.basecmd, d) - runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) - ud.repochanged = True - - def build_mirror_data(self, url, ud, d): - # Generate a mirror tarball if needed - if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): - os.chdir(ud.clonedir) - logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d) - - def unpack(self, ud, destdir, d): - """ unpack the downloaded src to destdir""" - - subdir = ud.parm.get("subpath", "") - if subdir != "": - readpathspec = ":%s" % (subdir) - else: - readpathspec = "" - - destdir = os.path.join(destdir, "git/") - if os.path.exists(destdir): - bb.utils.prunedir(destdir) - - runfetchcmd("git clone -s -n %s %s" % (ud.clonedir, destdir), d) - if not ud.nocheckout: - os.chdir(destdir) - runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d) - runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d) - return True - - def clean(self, ud, d): - """ clean the git directory """ - - bb.utils.remove(ud.localpath, True) - bb.utils.remove(ud.fullmirror) - - def supports_srcrev(self): - return True - - def _contains_ref(self, tag, d): - basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True) - return output.split()[0] != "0" - - def _revision_key(self, url, ud, d, name): - """ - Return a unique key for the url - """ - return "git:" + ud.host + ud.path.replace('/', '.') + ud.branches[name] - - def _latest_revision(self, url, ud, d, name): - """ - Compute the HEAD revision for the url - """ - if ud.user: - username = ud.user + '@' - else: - username = "" - - bb.fetch2.check_network_access(d, "git ls-remote %s%s %s" % (ud.host, ud.path, ud.branches[name])) - basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branches[name]) - output = runfetchcmd(cmd, d, True) - if not output: - raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, url) - return output.split()[0] - - def _build_revision(self, url, ud, d, name): - return ud.revisions[name] - - def _sortable_buildindex_disabled(self, url, ud, d, rev): - """ - Return a suitable buildindex for the revision specified. This is done by counting revisions - using "git rev-list" which may or may not work in different circumstances. - """ - - cwd = os.getcwd() - - # Check if we have the rev already - - if not os.path.exists(ud.clonedir): - print("no repo") - self.download(None, ud, d) - if not os.path.exists(ud.clonedir): - logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir) - return None - - - os.chdir(ud.clonedir) - if not self._contains_ref(rev, d): - self.download(None, ud, d) - - output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True) - os.chdir(cwd) - - buildindex = "%s" % output.split()[0] - logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev) - return buildindex diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py deleted file mode 100644 index 6a56f8d0c..000000000 --- a/bitbake/lib/bb/fetch2/hg.py +++ /dev/null @@ -1,174 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementation for mercurial DRCS (hg). - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# Copyright (C) 2004 Marcin Juszkiewicz -# Copyright (C) 2007 Robert Schuster -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import MissingParameterError -from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger - -class Hg(FetchMethod): - """Class to fetch from mercurial repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with mercurial. - """ - return ud.type in ['hg'] - - def urldata_init(self, ud, d): - """ - init hg specific variable within url data - """ - if not "module" in ud.parm: - raise MissingParameterError('module', ud.url) - - ud.module = ud.parm["module"] - - # Create paths to mercurial checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - elif not ud.revision: - ud.revision = self.latest_revision(ud.url, ud, d) - - ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) - - def need_update(self, url, ud, d): - revTag = ud.parm.get('rev', 'tip') - if revTag == "tip": - return True - if not os.path.exists(ud.localpath): - return True - return False - - def _buildhgcommand(self, ud, d, command): - """ - Build up an hg commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_hg}', d) - - proto = ud.parm.get('proto', 'http') - - host = ud.host - if proto == "file": - host = "/" - ud.host = "localhost" - - if not ud.user: - hgroot = host + ud.path - else: - hgroot = ud.user + "@" + host + ud.path - - if command is "info": - return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module) - - options = []; - if ud.revision: - options.append("-r %s" % ud.revision) - - if command is "fetch": - cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module) - elif command is "pull": - # do not pass options list; limiting pull to rev causes the local - # repo not to contain it and immediately following "update" command - # will crash - cmd = "%s pull" % (basecmd) - elif command is "update": - cmd = "%s update -C %s" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid hg command %s" % command, ud.url) - - return cmd - - def download(self, loc, ud, d): - """Fetch url""" - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): - updatecmd = self._buildhgcommand(ud, d, "pull") - logger.info("Update " + loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", updatecmd) - bb.fetch2.check_network_access(d, updatecmd, ud.url) - runfetchcmd(updatecmd, d) - - else: - fetchcmd = self._buildhgcommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", fetchcmd) - bb.fetch2.check_network_access(d, fetchcmd, ud.url) - runfetchcmd(fetchcmd, d) - - # Even when we clone (fetch), we still need to update as hg's clone - # won't checkout the specified revision if its on a branch - updatecmd = self._buildhgcommand(ud, d, "update") - os.chdir(ud.moddir) - logger.debug(1, "Running %s", updatecmd) - runfetchcmd(updatecmd, d) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.hg' --exclude '.hgrags'" - - os.chdir(ud.pkgdir) - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath]) - - def supports_srcrev(self): - return True - - def _latest_revision(self, url, ud, d, name): - """ - Compute tip revision for the url - """ - bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info")) - output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d) - return output.strip() - - def _build_revision(self, url, ud, d): - return ud.revision - - def _revision_key(self, url, ud, d, name): - """ - Return a unique key for the url - """ - return "hg:" + ud.moddir diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py deleted file mode 100644 index 77a296ec6..000000000 --- a/bitbake/lib/bb/fetch2/local.py +++ /dev/null @@ -1,80 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import bb -import bb.utils -from bb import data -from bb.fetch2 import FetchMethod - -class Local(FetchMethod): - def supports(self, url, urldata, d): - """ - Check to see if a given url represents a local fetch. - """ - return urldata.type in ['file'] - - def urldata_init(self, ud, d): - # We don't set localfile as for this fetcher the file is already local! - return - - def localpath(self, url, urldata, d): - """ - Return the local filename of a given url assuming a successful fetch. - """ - path = url.split("://")[1] - path = path.split(";")[0] - newpath = path - if path[0] != "/": - filespath = data.getVar('FILESPATH', d, True) - if filespath: - newpath = bb.utils.which(filespath, path) - if not newpath: - filesdir = data.getVar('FILESDIR', d, True) - if filesdir: - newpath = os.path.join(filesdir, path) - return newpath - - def download(self, url, urldata, d): - """Fetch urls (no-op for Local method)""" - # no need to fetch local files, we'll deal with them in place. - return 1 - - def checkstatus(self, url, urldata, d): - """ - Check the status of the url - """ - if urldata.localpath.find("*") != -1: - logger.info("URL %s looks like a glob and was therefore not checked.", url) - return True - if os.path.exists(urldata.localpath): - return True - return False - - def clean(self, urldata, d): - return - diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py deleted file mode 100644 index 4bf411c24..000000000 --- a/bitbake/lib/bb/fetch2/osc.py +++ /dev/null @@ -1,135 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -Bitbake "Fetch" implementation for osc (Opensuse build service client). -Based on the svn "Fetch" implementation. - -""" - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import MissingParameterError -from bb.fetch2 import runfetchcmd - -class Osc(FetchMethod): - """Class to fetch a module or modules from Opensuse build server - repositories.""" - - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with osc. - """ - return ud.type in ['osc'] - - def urldata_init(self, ud, d): - if not "module" in ud.parm: - raise MissingParameterError('module', ud.url) - - ud.module = ud.parm["module"] - - # Create paths to osc checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host) - ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - else: - pv = data.getVar("PV", d, 0) - rev = bb.fetch2.srcrev_internal_helper(ud, d) - if rev and rev != True: - ud.revision = rev - else: - ud.revision = "" - - ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) - - def _buildosccommand(self, ud, d, command): - """ - Build up an ocs commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_osc}', d) - - proto = ud.parm.get('proto', 'ocs') - - options = [] - - config = "-c %s" % self.generate_config(ud, d) - - if ud.revision: - options.append("-r %s" % ud.revision) - - coroot = self._strip_leading_slashes(ud.path) - - if command is "fetch": - osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) - elif command is "update": - osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) - else: - raise FetchError("Invalid osc command %s" % command, ud.url) - - return osccmd - - def download(self, loc, ud, d): - """ - Fetch url - """ - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): - oscupdatecmd = self._buildosccommand(ud, d, "update") - logger.info("Update "+ loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", oscupdatecmd) - bb.fetch2.check_network_access(d, oscupdatecmd, ud.url) - runfetchcmd(oscupdatecmd, d) - else: - oscfetchcmd = self._buildosccommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", oscfetchcmd) - bb.fetch2.check_network_access(d, oscfetchcmd, ud.url) - runfetchcmd(oscfetchcmd, d) - - os.chdir(os.path.join(ud.pkgdir + ud.path)) - # tar them up to a defined filename - runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath]) - - def supports_srcrev(self): - return False - - def generate_config(self, ud, d): - """ - Generate a .oscrc to be used for this run. - """ - - config_path = os.path.join(data.expand('${OSCDIR}', d), "oscrc") - if (os.path.exists(config_path)): - os.remove(config_path) - - f = open(config_path, 'w') - f.write("[general]\n") - f.write("apisrv = %s\n" % ud.host) - f.write("scheme = http\n") - f.write("su-wrapper = su -c\n") - f.write("build-root = %s\n" % data.expand('${WORKDIR}', d)) - f.write("urllist = http://moblin-obs.jf.intel.com:8888/build/%(project)s/%(repository)s/%(buildarch)s/:full/%(name)s.rpm\n") - f.write("extra-pkgs = gzip\n") - f.write("\n") - f.write("[%s]\n" % ud.host) - f.write("user = %s\n" % ud.parm["user"]) - f.write("pass = %s\n" % ud.parm["pswd"]) - f.close() - - return config_path diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py deleted file mode 100644 index 6347834c7..000000000 --- a/bitbake/lib/bb/fetch2/perforce.py +++ /dev/null @@ -1,196 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -from future_builtins import zip -import os -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import logger -from bb.fetch2 import runfetchcmd - -class Perforce(FetchMethod): - def supports(self, url, ud, d): - return ud.type in ['p4'] - - def doparse(url, d): - parm = {} - path = url.split("://")[1] - delim = path.find("@"); - if delim != -1: - (user, pswd, host, port) = path.split('@')[0].split(":") - path = path.split('@')[1] - else: - (host, port) = data.getVar('P4PORT', d).split(':') - user = "" - pswd = "" - - if path.find(";") != -1: - keys=[] - values=[] - plist = path.split(';') - for item in plist: - if item.count('='): - (key, value) = item.split('=') - keys.append(key) - values.append(value) - - parm = dict(zip(keys, values)) - path = "//" + path.split(';')[0] - host += ":%s" % (port) - parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm) - - return host, path, user, pswd, parm - doparse = staticmethod(doparse) - - def getcset(d, depot, host, user, pswd, parm): - p4opt = "" - if "cset" in parm: - return parm["cset"]; - if user: - p4opt += " -u %s" % (user) - if pswd: - p4opt += " -P %s" % (pswd) - if host: - p4opt += " -p %s" % (host) - - p4date = data.getVar("P4DATE", d, True) - if "revision" in parm: - depot += "#%s" % (parm["revision"]) - elif "label" in parm: - depot += "@%s" % (parm["label"]) - elif p4date: - depot += "@%s" % (p4date) - - p4cmd = data.getVar('FETCHCOMMAND_p4', d, True) - logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot) - p4file = os.popen("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot)) - cset = p4file.readline().strip() - logger.debug(1, "READ %s", cset) - if not cset: - return -1 - - return cset.split(' ')[1] - getcset = staticmethod(getcset) - - def urldata_init(self, ud, d): - (host, path, user, pswd, parm) = Perforce.doparse(ud.url, d) - - # If a label is specified, we use that as our filename - - if "label" in parm: - ud.localfile = "%s.tar.gz" % (parm["label"]) - return - - base = path - which = path.find('/...') - if which != -1: - base = path[:which] - - base = self._strip_leading_slashes(base) - - cset = Perforce.getcset(d, path, host, user, pswd, parm) - - ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d) - - def download(self, loc, ud, d): - """ - Fetch urls - """ - - (host, depot, user, pswd, parm) = Perforce.doparse(loc, d) - - if depot.find('/...') != -1: - path = depot[:depot.find('/...')] - else: - path = depot - - module = parm.get('module', os.path.basename(path)) - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - # Get the p4 command - p4opt = "" - if user: - p4opt += " -u %s" % (user) - - if pswd: - p4opt += " -P %s" % (pswd) - - if host: - p4opt += " -p %s" % (host) - - p4cmd = data.getVar('FETCHCOMMAND', localdata, True) - - # create temp directory - logger.debug(2, "Fetch: creating temporary directory") - bb.mkdirhier(data.expand('${WORKDIR}', localdata)) - data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata) - tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, True) or "false") - tmpfile = tmppipe.readline().strip() - if not tmpfile: - raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc) - - if "label" in parm: - depot = "%s@%s" % (depot, parm["label"]) - else: - cset = Perforce.getcset(d, depot, host, user, pswd, parm) - depot = "%s@%s" % (depot, cset) - - os.chdir(tmpfile) - logger.info("Fetch " + loc) - logger.info("%s%s files %s", p4cmd, p4opt, depot) - p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot)) - - if not p4file: - raise FetchError("Fetch: unable to get the P4 files from %s" % depot, loc) - - count = 0 - - for file in p4file: - list = file.split() - - if list[2] == "delete": - continue - - dest = list[0][len(path)+1:] - where = dest.find("#") - - os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0])) - count = count + 1 - - if count == 0: - logger.error() - raise FetchError("Fetch: No files gathered from the P4 fetch", loc) - - runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath]) - # cleanup - bb.utils.prunedir(tmpfile) diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py deleted file mode 100644 index 54130a8c3..000000000 --- a/bitbake/lib/bb/fetch2/repo.py +++ /dev/null @@ -1,98 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake "Fetch" repo (git) implementation - -""" - -# Copyright (C) 2009 Tom Rini <trini@embeddedalley.com> -# -# Based on git.py which is: -#Copyright (C) 2005 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import runfetchcmd - -class Repo(FetchMethod): - """Class to fetch a module or modules from repo (git) repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with repo. - """ - return ud.type in ["repo"] - - def urldata_init(self, ud, d): - """ - We don"t care about the git rev of the manifests repository, but - we do care about the manifest to use. The default is "default". - We also care about the branch or tag to be used. The default is - "master". - """ - - ud.proto = ud.parm.get('protocol', 'git') - ud.branch = ud.parm.get('branch', 'master') - ud.manifest = ud.parm.get('manifest', 'default.xml') - if not ud.manifest.endswith('.xml'): - ud.manifest += '.xml' - - ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) - - def download(self, loc, ud, d): - """Fetch url""" - - if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): - logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) - return - - gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) - repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") - codir = os.path.join(repodir, gitsrcname, ud.manifest) - - if ud.user: - username = ud.user + "@" - else: - username = "" - - bb.mkdirhier(os.path.join(codir, "repo")) - os.chdir(os.path.join(codir, "repo")) - if not os.path.exists(os.path.join(codir, "repo", ".repo")): - bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url) - runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) - - bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url) - runfetchcmd("repo sync", d) - os.chdir(codir) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.repo' --exclude '.git'" - - # Create a cache - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d) - - def supports_srcrev(self): - return False - - def _build_revision(self, url, ud, d): - return ud.manifest - - def _want_sortable_revision(self, url, ud, d): - return False diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py deleted file mode 100644 index 91ac15faa..000000000 --- a/bitbake/lib/bb/fetch2/ssh.py +++ /dev/null @@ -1,120 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -''' -BitBake 'Fetch' implementations - -This implementation is for Secure Shell (SSH), and attempts to comply with the -IETF secsh internet draft: - http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/ - - Currently does not support the sftp parameters, as this uses scp - Also does not support the 'fingerprint' connection parameter. - -''' - -# Copyright (C) 2006 OpenedHand Ltd. -# -# -# Based in part on svk.py: -# Copyright (C) 2006 Holger Hans Peter Freyther -# Based on svn.py: -# Copyright (C) 2003, 2004 Chris Larson -# Based on functions from the base bb module: -# Copyright 2003 Holger Schurig -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import re, os -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import logger -from bb.fetch2 import runfetchcmd - - -__pattern__ = re.compile(r''' - \s* # Skip leading whitespace - ssh:// # scheme - ( # Optional username/password block - (?P<user>\S+) # username - (:(?P<pass>\S+))? # colon followed by the password (optional) - )? - (?P<cparam>(;[^;]+)*)? # connection parameters block (optional) - @ - (?P<host>\S+?) # non-greedy match of the host - (:(?P<port>[0-9]+))? # colon followed by the port (optional) - / - (?P<path>[^;]+) # path on the remote system, may be absolute or relative, - # and may include the use of '~' to reference the remote home - # directory - (?P<sparam>(;[^;]+)*)? # parameters block (optional) - $ -''', re.VERBOSE) - -class SSH(FetchMethod): - '''Class to fetch a module or modules via Secure Shell''' - - def supports(self, url, urldata, d): - return __pattern__.match(url) != None - - def localpath(self, url, urldata, d): - m = __pattern__.match(urldata.url) - path = m.group('path') - host = m.group('host') - lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path)) - return lpath - - def download(self, url, urldata, d): - dldir = data.getVar('DL_DIR', d, True) - - m = __pattern__.match(url) - path = m.group('path') - host = m.group('host') - port = m.group('port') - user = m.group('user') - password = m.group('pass') - - ldir = os.path.join(dldir, host) - lpath = os.path.join(ldir, os.path.basename(path)) - - if not os.path.exists(ldir): - os.makedirs(ldir) - - if port: - port = '-P %s' % port - else: - port = '' - - if user: - fr = user - if password: - fr += ':%s' % password - fr += '@%s' % host - else: - fr = host - fr += ':%s' % path - - - import commands - cmd = 'scp -B -r %s %s %s/' % ( - port, - commands.mkarg(fr), - commands.mkarg(ldir) - ) - - bb.fetch2.check_network_access(d, cmd, urldata.url) - - runfetchcmd(cmd, d) - diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py deleted file mode 100644 index 6211cac8d..000000000 --- a/bitbake/lib/bb/fetch2/svk.py +++ /dev/null @@ -1,97 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -This implementation is for svk. It is based on the svn implementation - -""" - -# Copyright (C) 2006 Holger Hans Peter Freyther -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import MissingParameterError -from bb.fetch2 import logger -from bb.fetch2 import runfetchcmd - -class Svk(FetchMethod): - """Class to fetch a module or modules from svk repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with svk. - """ - return ud.type in ['svk'] - - def urldata_init(self, ud, d): - - if not "module" in ud.parm: - raise MissingParameterError('module', ud.url) - else: - ud.module = ud.parm["module"] - - ud.revision = ud.parm.get('rev', "") - - ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) - - def need_update(self, url, ud, d): - if ud.date == "now": - return True - if not os.path.exists(ud.localpath): - return True - return False - - def download(self, loc, ud, d): - """Fetch urls""" - - svkroot = ud.host + ud.path - - svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) - - if ud.revision: - svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module) - - # create temp directory - localdata = data.createCopy(d) - data.update_data(localdata) - logger.debug(2, "Fetch: creating temporary directory") - bb.mkdirhier(data.expand('${WORKDIR}', localdata)) - data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) - tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, True) or "false") - tmpfile = tmppipe.readline().strip() - if not tmpfile: - logger.error() - raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc) - - # check out sources there - os.chdir(tmpfile) - logger.info("Fetch " + loc) - logger.debug(1, "Running %s", svkcmd) - runfetchcmd(svkcmd, d, cleanup = [tmpfile]) - - os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) - # tar them up to a defined filename - runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d, cleanup = [ud.localpath]) - - # cleanup - bb.utils.prunedir(tmpfile) diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py deleted file mode 100644 index ac4fd27e1..000000000 --- a/bitbake/lib/bb/fetch2/svn.py +++ /dev/null @@ -1,180 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementation for svn. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# Copyright (C) 2004 Marcin Juszkiewicz -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import MissingParameterError -from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger - -class Svn(FetchMethod): - """Class to fetch a module or modules from svn repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with svn. - """ - return ud.type in ['svn'] - - def urldata_init(self, ud, d): - """ - init svn specific variable within url data - """ - if not "module" in ud.parm: - raise MissingParameterError('module', ud.url) - - ud.module = ud.parm["module"] - - # Create paths to svn checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - - ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) - - def _buildsvncommand(self, ud, d, command): - """ - Build up an svn commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_svn}', d) - - proto = ud.parm.get('proto', 'svn') - - svn_rsh = None - if proto == "svn+ssh" and "rsh" in ud.parm: - svn_rsh = ud.parm["rsh"] - - svnroot = ud.host + ud.path - - options = [] - - if ud.user: - options.append("--username %s" % ud.user) - - if ud.pswd: - options.append("--password %s" % ud.pswd) - - if command is "info": - svncmd = "%s info %s %s://%s/%s/" % (basecmd, " ".join(options), proto, svnroot, ud.module) - else: - suffix = "" - if ud.revision: - options.append("-r %s" % ud.revision) - suffix = "@%s" % (ud.revision) - - if command is "fetch": - svncmd = "%s co %s %s://%s/%s%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, suffix, ud.module) - elif command is "update": - svncmd = "%s update %s" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid svn command %s" % command, ud.url) - - if svn_rsh: - svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) - - return svncmd - - def download(self, loc, ud, d): - """Fetch url""" - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): - svnupdatecmd = self._buildsvncommand(ud, d, "update") - logger.info("Update " + loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", svnupdatecmd) - bb.fetch2.check_network_access(d, svnupdatecmd, ud.url) - runfetchcmd(svnupdatecmd, d) - else: - svnfetchcmd = self._buildsvncommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", svnfetchcmd) - bb.fetch2.check_network_access(d, svnfetchcmd, ud.url) - runfetchcmd(svnfetchcmd, d) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.svn'" - - os.chdir(ud.pkgdir) - # tar them up to a defined filename - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath]) - - def clean(self, ud, d): - """ Clean SVN specific files and dirs """ - - bb.utils.remove(ud.localpath) - bb.utils.remove(ud.moddir, True) - - - def supports_srcrev(self): - return True - - def _revision_key(self, url, ud, d, name): - """ - Return a unique key for the url - """ - return "svn:" + ud.moddir - - def _latest_revision(self, url, ud, d, name): - """ - Return the latest upstream revision number - """ - bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "info")) - - output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True) - - revision = None - for line in output.splitlines(): - if "Last Changed Rev" in line: - revision = line.split(":")[1].strip() - - return revision - - def _sortable_revision(self, url, ud, d): - """ - Return a sortable revision number which in our case is the revision number - """ - - return self._build_revision(url, ud, d) - - def _build_revision(self, url, ud, d): - return ud.revision diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py deleted file mode 100644 index 7bd027adc..000000000 --- a/bitbake/lib/bb/fetch2/wget.py +++ /dev/null @@ -1,91 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import logging -import bb -import urllib -from bb import data -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import encodeurl -from bb.fetch2 import decodeurl -from bb.fetch2 import logger -from bb.fetch2 import runfetchcmd - -class Wget(FetchMethod): - """Class to fetch urls via 'wget'""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with wget. - """ - return ud.type in ['http', 'https', 'ftp'] - - def urldata_init(self, ud, d): - - ud.basename = os.path.basename(ud.path) - ud.localfile = data.expand(urllib.unquote(ud.basename), d) - - def download(self, uri, ud, d, checkonly = False): - """Fetch urls""" - - def fetch_uri(uri, ud, d): - if checkonly: - fetchcmd = data.getVar("CHECKCOMMAND", d, True) - elif os.path.exists(ud.localpath): - # file exists, but we didnt complete it.. trying again.. - fetchcmd = data.getVar("RESUMECOMMAND", d, True) - else: - fetchcmd = data.getVar("FETCHCOMMAND", d, True) - - uri = uri.split(";")[0] - uri_decoded = list(decodeurl(uri)) - uri_type = uri_decoded[0] - uri_host = uri_decoded[1] - - fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0]) - fetchcmd = fetchcmd.replace("${FILE}", ud.basename) - logger.info("fetch " + uri) - logger.debug(2, "executing " + fetchcmd) - bb.fetch2.check_network_access(d, fetchcmd) - runfetchcmd(fetchcmd, d) - - # Sanity check since wget can pretend it succeed when it didn't - # Also, this used to happen if sourceforge sent us to the mirror page - if not os.path.exists(ud.localpath) and not checkonly: - raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - fetch_uri(uri, ud, localdata) - - return True - - def checkstatus(self, uri, ud, d): - return self.download(uri, ud, d, True) |